summaryrefslogtreecommitdiffstats
path: root/searchcore
diff options
context:
space:
mode:
authorTor Egge <Tor.Egge@yahoo-inc.com>2017-04-21 15:13:05 +0000
committerTor Egge <Tor.Egge@yahoo-inc.com>2017-04-24 09:28:11 +0000
commitc9429079a48d8cc03039f811abd456e9865911b1 (patch)
tree90d12a7f72900ed429be12a19d7ed3d220ac1d89 /searchcore
parentc2a6df2cd34f8a9f989d47f422939a7ab6874c91 (diff)
Remove history schema and wiping of old removed fields.
Diffstat (limited to 'searchcore')
-rw-r--r--searchcore/src/apps/tests/persistenceconformance_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp9
-rw-r--r--searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp141
-rw-r--r--searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp86
-rw-r--r--searchcore/src/tests/proton/server/feedstates_test.cpp1
-rw-r--r--searchcore/src/tests/proton/server/memoryconfigstore_test.cpp110
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt1
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/configstore.h6
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h9
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.cpp140
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.h17
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/feedstates.cpp6
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp73
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h14
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h16
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp38
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h6
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp28
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h28
25 files changed, 51 insertions, 699 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp
index 75c37394e83..a0bb3cf8591 100644
--- a/searchcore/src/apps/tests/persistenceconformance_test.cpp
+++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp
@@ -164,7 +164,7 @@ public:
vespalib::string inputCfg = _baseDir + "/" + docType.toString() + "/baseconfig";
{
FileConfigManager fileCfg(inputCfg, "", docType.getName());
- fileCfg.saveConfig(*snapshot, Schema(), 1);
+ fileCfg.saveConfig(*snapshot, 1);
}
config::DirSpec spec(inputCfg + "/config-1");
TuneFileDocumentDB::SP tuneFileDocDB(new TuneFileDocumentDB());
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
index 5e3b576cf9c..63ec80e1f8a 100644
--- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
@@ -139,7 +139,6 @@ struct MyResourceWriteFilter : public IResourceWriteFilter
struct MyReplayConfig : public IReplayConfig {
virtual void replayConfig(SerialNum) override {}
- virtual void replayWipeHistory(SerialNum, fastos::TimeStamp) override {}
};
void ackToken(FeedToken *token) {
@@ -480,17 +479,13 @@ struct FeedHandlerFixture
struct MyConfigStore : ConfigStore {
virtual SerialNum getBestSerialNum() const override { return 1; }
virtual SerialNum getOldestSerialNum() const override { return 1; }
- virtual void saveConfig(const DocumentDBConfig &,
- const search::index::Schema &, SerialNum) override {}
+ virtual void saveConfig(const DocumentDBConfig &, SerialNum) override {}
virtual void loadConfig(const DocumentDBConfig &, SerialNum,
- DocumentDBConfig::SP &,
- search::index::Schema::SP &) override {}
+ DocumentDBConfig::SP &) override {}
virtual void removeInvalid() override {}
void prune(SerialNum) override {}
virtual bool hasValidSerial(SerialNum) const override { return true; }
virtual SerialNum getPrevValidSerial(SerialNum) const override { return 1; }
- virtual void saveWipeHistoryConfig(SerialNum,
- fastos::TimeStamp) override {}
virtual void serializeConfig(SerialNum, vespalib::nbostream &) override {}
virtual void deserializeConfig(SerialNum, vespalib::nbostream &) override {}
virtual void setProtonConfig(const ProtonConfigSP &) override { }
diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
index 579e33250b3..4fd2f7f0332 100644
--- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp
@@ -32,31 +32,6 @@ using vespalib::nbostream;
vespalib::string myId("myconfigid");
-namespace
-{
-
-DocumentDBConfig::SP
-getConfig(int64_t generation, const Schema::SP &schema)
-{
- return test::DocumentDBConfigBuilder(generation, schema, "client", "test").build();
-}
-
-Schema::SP
-getSchema(int step)
-{
- Schema::SP schema(new Schema);
- schema->addIndexField(Schema::IndexField("foo1", schema::DataType::STRING));
- if (step < 2) {
- schema->addIndexField(Schema::IndexField("foo2", schema::DataType::STRING));
- }
- if (step < 1) {
- schema->addIndexField(Schema::IndexField("foo3", schema::DataType::STRING));
- }
- return schema;
-}
-
- }
-
DocumentDBConfig::SP
makeBaseConfigSnapshot()
{
@@ -79,19 +54,11 @@ makeBaseConfigSnapshot()
return snap;
}
-Schema
-makeHistorySchema()
-{
- Schema hs;
- hs.addIndexField(Schema::IndexField("history", schema::DataType::STRING));
- return hs;
-}
-
void
-saveBaseConfigSnapshot(const DocumentDBConfig &snap, const Schema &history, SerialNum num)
+saveBaseConfigSnapshot(const DocumentDBConfig &snap, SerialNum num)
{
FileConfigManager cm("out", myId, snap.getDocTypeName());
- cm.saveConfig(snap, history, num);
+ cm.saveConfig(snap, num);
}
@@ -157,26 +124,22 @@ addConfigsThatAreNotSavedToDisk(const DocumentDBConfig &cfg)
return builder.build();
}
-TEST_FF("requireThatConfigCanBeSavedAndLoaded", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
- Schema(makeHistorySchema()))
+TEST_F("requireThatConfigCanBeSavedAndLoaded", DocumentDBConfig::SP(makeBaseConfigSnapshot()))
{
- DocumentDBConfig::SP fullCfg = addConfigsThatAreNotSavedToDisk(*f1);
- saveBaseConfigSnapshot(*fullCfg, f2, 20);
+ DocumentDBConfig::SP fullCfg = addConfigsThatAreNotSavedToDisk(*f);
+ saveBaseConfigSnapshot(*fullCfg, 20);
DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot());
- Schema::SP ehs;
{
FileConfigManager cm("out", myId, "dummy");
- cm.loadConfig(*esnap, 20, esnap, ehs);
+ cm.loadConfig(*esnap, 20, esnap);
}
- assertEqualSnapshot(*f1, *esnap);
- EXPECT_TRUE(f2 == *ehs);
+ assertEqualSnapshot(*f, *esnap);
}
-TEST_FF("requireThatConfigCanBeSerializedAndDeserialized", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
- Schema(makeHistorySchema()))
+TEST_F("requireThatConfigCanBeSerializedAndDeserialized", DocumentDBConfig::SP(makeBaseConfigSnapshot()))
{
- saveBaseConfigSnapshot(*f1, f2, 30);
+ saveBaseConfigSnapshot(*f, 30);
nbostream stream;
{
FileConfigManager cm("out", myId, "dummy");
@@ -187,100 +150,32 @@ TEST_FF("requireThatConfigCanBeSerializedAndDeserialized", DocumentDBConfig::SP(
cm.deserializeConfig(40, stream);
}
DocumentDBConfig::SP fsnap(makeEmptyConfigSnapshot());
- Schema::SP fhs;
{
FileConfigManager cm("out", myId, "dummy");
- cm.loadConfig(*fsnap, 40, fsnap, fhs);
+ cm.loadConfig(*fsnap, 40, fsnap);
}
- assertEqualSnapshot(*f1, *fsnap);
- EXPECT_TRUE(f2 == *fhs);
+ assertEqualSnapshot(*f, *fsnap);
EXPECT_EQUAL("dummy", fsnap->getDocTypeName());
}
-TEST_FF("requireThatWipeHistoryCanBeSaved", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
- Schema(makeHistorySchema()))
-{
- saveBaseConfigSnapshot(*f1, f2, 50);
- {
- FileConfigManager cm("out", myId, "dummy");
- cm.saveWipeHistoryConfig(60, 0);
- }
- DocumentDBConfig::SP gsnap(makeEmptyConfigSnapshot());
- Schema::SP ghs;
- {
- FileConfigManager cm("out", myId, "dummy");
- cm.loadConfig(*gsnap, 60, gsnap, ghs);
- }
- assertEqualSnapshot(*f1, *gsnap);
- EXPECT_TRUE(f2 != *ghs);
- EXPECT_TRUE(!f2.empty());
- EXPECT_TRUE(ghs->empty());
-}
-
-
-TEST("require that wipe history clears only portions of history")
-{
- FileConfigManager cm("out2", myId, "dummy");
- Schema::SP schema(getSchema(0));
- Schema::SP history(new Schema);
- DocumentDBConfig::SP config(getConfig(5, schema));
- cm.saveConfig(*config, *history, 5);
- Schema::SP oldSchema(schema);
- schema = getSchema(1);
- config = getConfig(6, schema);
- history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
- 100);
- cm.saveConfig(*config, *history, 10);
- oldSchema = schema;
- schema = getSchema(2);
- config = getConfig(7, schema);
- history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
- 200);
- cm.saveConfig(*config, *history, 15);
- cm.saveWipeHistoryConfig(20, 50);
- cm.saveWipeHistoryConfig(25, 100);
- cm.saveWipeHistoryConfig(30, 150);
- cm.saveWipeHistoryConfig(35, 200);
- cm.saveWipeHistoryConfig(40, 250);
- DocumentDBConfig::SP oldconfig(config);
- cm.loadConfig(*oldconfig, 20, config, history);
- EXPECT_EQUAL(2u, history->getNumIndexFields());
- oldconfig = config;
- cm.loadConfig(*oldconfig, 25, config, history);
- EXPECT_EQUAL(2u, history->getNumIndexFields());
- oldconfig = config;
- cm.loadConfig(*oldconfig, 30, config, history);
- EXPECT_EQUAL(1u, history->getNumIndexFields());
- oldconfig = config;
- cm.loadConfig(*oldconfig, 35, config, history);
- EXPECT_EQUAL(1u, history->getNumIndexFields());
- oldconfig = config;
- cm.loadConfig(*oldconfig, 40, config, history);
- EXPECT_EQUAL(0u, history->getNumIndexFields());
-}
-
-TEST_FF("requireThatConfigCanBeLoadedWithoutExtraConfigsDataFile", DocumentDBConfig::SP(makeBaseConfigSnapshot()),
- Schema(makeHistorySchema()))
+TEST_F("requireThatConfigCanBeLoadedWithoutExtraConfigsDataFile", DocumentDBConfig::SP(makeBaseConfigSnapshot()))
{
- saveBaseConfigSnapshot(*f1, f2, 70);
+ saveBaseConfigSnapshot(*f, 70);
EXPECT_TRUE(vespalib::unlink("out/config-70/extraconfigs.dat"));
DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot());
- Schema::SP ehs;
{
FileConfigManager cm("out", myId, "dummy");
- cm.loadConfig(*esnap, 70, esnap, ehs);
+ cm.loadConfig(*esnap, 70, esnap);
}
EXPECT_EQUAL(0u, esnap->getExtraConfigs().size());
}
-TEST_FF("requireThatVisibilityDelayIsPropagated",
- DocumentDBConfig::SP(makeBaseConfigSnapshot()),
- Schema(makeHistorySchema()))
+TEST_F("requireThatVisibilityDelayIsPropagated",
+ DocumentDBConfig::SP(makeBaseConfigSnapshot()))
{
- saveBaseConfigSnapshot(*f1, f2, 80);
+ saveBaseConfigSnapshot(*f, 80);
DocumentDBConfig::SP esnap(makeEmptyConfigSnapshot());
- Schema::SP ehs;
{
ProtonConfigBuilder protonConfigBuilder;
ProtonConfigBuilder::Documentdb ddb;
@@ -292,7 +187,7 @@ TEST_FF("requireThatVisibilityDelayIsPropagated",
using ProtonConfigSP = BootstrapConfig::ProtonConfigSP;
cm.setProtonConfig(
ProtonConfigSP(new ProtonConfig(protonConfigBuilder)));
- cm.loadConfig(*esnap, 70, esnap, ehs);
+ cm.loadConfig(*esnap, 70, esnap);
}
EXPECT_EQUAL(0u, esnap->getExtraConfigs().size());
EXPECT_EQUAL(61.0, esnap->getMaintenanceConfigSP()->getVisibilityDelay().sec());
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index 4ed6953ca14..2605beadf3b 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -195,14 +195,12 @@ struct MySessionCachePruner : public ISessionCachePruner
class MyFeedHandler : public IDocumentMoveHandler,
public IPruneRemovedDocumentsHandler,
public IHeartBeatHandler,
- public IWipeOldRemovedFieldsHandler,
public IOperationStorer
{
FastOS_ThreadId _executorThreadId;
std::vector<MyDocumentSubDB *> _subDBs;
SerialNum _serialNum;
uint32_t _heartBeats;
- fastos::TimeStamp _wipeTimeLimit;
public:
MyFeedHandler(FastOS_ThreadId &executorThreadId);
@@ -221,9 +219,6 @@ public:
virtual void
heartBeat() override;
- virtual void
- wipeOldRemovedFields(TimeStamp wipeTimeLimit) override;
-
void
setSubDBs(const std::vector<MyDocumentSubDB *> &subDBs);
@@ -242,12 +237,6 @@ public:
{
return _heartBeats;
}
-
- fastos::TimeStamp
- getWipeTimeLimit()
- {
- return _wipeTimeLimit;
- }
};
@@ -499,7 +488,6 @@ public:
newCfg(new DocumentDBMaintenanceConfig(
pruneConfig,
_mcCfg->getHeartBeatConfig(),
- _mcCfg->getWipeOldRemovedFieldsConfig(),
_mcCfg->getSessionCachePruneInterval(),
_mcCfg->getVisibilityDelay(),
_mcCfg->getLidSpaceCompactionConfig(),
@@ -517,7 +505,6 @@ public:
newCfg(new DocumentDBMaintenanceConfig(
_mcCfg->getPruneRemovedDocumentsConfig(),
heartBeatConfig,
- _mcCfg->getWipeOldRemovedFieldsConfig(),
_mcCfg->getSessionCachePruneInterval(),
_mcCfg->getVisibilityDelay(),
_mcCfg->getLidSpaceCompactionConfig(),
@@ -529,32 +516,12 @@ public:
}
void
- setWipeOldRemovedFieldsConfig(const DocumentDBWipeOldRemovedFieldsConfig &wipeConfig)
- {
- DocumentDBMaintenanceConfig::SP
- newCfg(new DocumentDBMaintenanceConfig(
- _mcCfg->getPruneRemovedDocumentsConfig(),
- _mcCfg->getHeartBeatConfig(),
- wipeConfig,
- _mcCfg->getSessionCachePruneInterval(),
- _mcCfg->getVisibilityDelay(),
- _mcCfg->getLidSpaceCompactionConfig(),
- _mcCfg->getAttributeUsageFilterConfig(),
- _mcCfg->getAttributeUsageSampleInterval(),
- _mcCfg->getResourceLimitFactor()));
- _mcCfg = newCfg;
- forwardMaintenanceConfig();
- }
-
-
- void
setGroupingSessionPruneInterval(double groupingSessionPruneInterval)
{
DocumentDBMaintenanceConfig::SP
newCfg(new DocumentDBMaintenanceConfig(
_mcCfg->getPruneRemovedDocumentsConfig(),
_mcCfg->getHeartBeatConfig(),
- _mcCfg->getWipeOldRemovedFieldsConfig(),
groupingSessionPruneInterval,
_mcCfg->getVisibilityDelay(),
_mcCfg->getLidSpaceCompactionConfig(),
@@ -570,7 +537,6 @@ public:
newCfg(new DocumentDBMaintenanceConfig(
_mcCfg->getPruneRemovedDocumentsConfig(),
_mcCfg->getHeartBeatConfig(),
- _mcCfg->getWipeOldRemovedFieldsConfig(),
_mcCfg->getSessionCachePruneInterval(),
_mcCfg->getVisibilityDelay(),
cfg,
@@ -790,8 +756,7 @@ MyFeedHandler::MyFeedHandler(FastOS_ThreadId &executorThreadId)
_executorThreadId(executorThreadId),
_subDBs(),
_serialNum(0u),
- _heartBeats(0u),
- _wipeTimeLimit()
+ _heartBeats(0u)
{
}
@@ -849,14 +814,6 @@ MyFeedHandler::heartBeat()
void
-MyFeedHandler::wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit)
-{
- assert(isExecutorThread());
- _wipeTimeLimit = wipeTimeLimit;
-}
-
-
-void
MyFeedHandler::setSubDBs(const std::vector<MyDocumentSubDB *> &subDBs)
{
_subDBs = subDBs;
@@ -1000,7 +957,7 @@ void
MaintenanceControllerFixture::injectMaintenanceJobs()
{
if (_injectDefaultJobs) {
- MaintenanceJobsInjector::injectJobs(_mc, *_mcCfg, _fh, _gsp, _fh,
+ MaintenanceJobsInjector::injectJobs(_mc, *_mcCfg, _fh, _gsp,
_lscHandlers, _fh, _mc, _docTypeName.getName(),
_fh, _fh, _bmc, _clusterStateHandler, _bucketHandler,
_calc,
@@ -1224,38 +1181,6 @@ TEST_F("require that periodic session prunings are scheduled",
ASSERT_TRUE(f._gsp.isInvoked);
}
-TEST_F("require that wipe old removed fields are scheduled",
- MaintenanceControllerFixture)
-{
- f.notifyClusterStateChanged();
- f.startMaintenance();
- TimeStamp now0 = TimeStamp(ClockSystem::now());
- f.setWipeOldRemovedFieldsConfig(DocumentDBWipeOldRemovedFieldsConfig(0.2, 100));
- TimeStamp now = TimeStamp(ClockSystem::now());
- TimeStamp expWipeTimeLimit = now - TimeStamp(100 * TimeStamp::SEC);
- TimeStamp wtLim;
- for (uint32_t i = 0; i < 600; ++i) {
- FastOS_Thread::Sleep(100);
- wtLim = f._fh.getWipeTimeLimit();
- if (wtLim.sec() != 0u) {
- break;
- }
- }
- TimeStamp now1 = TimeStamp(ClockSystem::now());
- double fuzz = now1.sec() - now0.sec();
- LOG(info,
- "WipeOldRemovedFields: "
- "now(%" PRIu64 "), "
- "expWipeTimeLimit(%" PRIu64 "), "
- "actWipeTimeLimit(%" PRIu64 "), "
- "fuzz(%05.3f)",
- (uint64_t)now.sec(),
- (uint64_t)expWipeTimeLimit.sec(),
- (uint64_t)wtLim.sec(),
- fuzz);
- EXPECT_APPROX(expWipeTimeLimit.sec(), wtLim.sec(), 4u + fuzz);
-}
-
TEST_F("require that active bucket is not moved until de-activated", MaintenanceControllerFixture)
{
f._builder.createDocs(1, 1, 4); // 3 docs
@@ -1477,13 +1402,13 @@ TEST_F("require that lid space compaction jobs can be disabled", MaintenanceCont
f.forwardMaintenanceConfig();
{
auto jobs = f._mc.getJobList();
- EXPECT_EQUAL(7u, jobs.size());
+ EXPECT_EQUAL(6u, jobs.size());
EXPECT_TRUE(containsJob(jobs, "lid_space_compaction.my_handler"));
}
f.setLidSpaceCompactionConfig(DocumentDBLidSpaceCompactionConfig::createDisabled());
{
auto jobs = f._mc.getJobList();
- EXPECT_EQUAL(6u, jobs.size());
+ EXPECT_EQUAL(5u, jobs.size());
EXPECT_FALSE(containsJob(jobs, "lid_space_compaction.my_handler"));
}
}
@@ -1492,10 +1417,9 @@ TEST_F("Require that maintenance jobs are run by correct executor", MaintenanceC
{
f.injectMaintenanceJobs();
auto jobs = f._mc.getJobList();
- EXPECT_EQUAL(6u, jobs.size());
+ EXPECT_EQUAL(5u, jobs.size());
EXPECT_TRUE(containsJobAndExecutedBy(jobs, "heart_beat", f._threadService));
EXPECT_TRUE(containsJobAndExecutedBy(jobs, "prune_session_cache", f._genericExecutor));
- EXPECT_TRUE(containsJobAndExecutedBy(jobs, "wipe_old_removed_fields", f._threadService));
EXPECT_TRUE(containsJobAndExecutedBy(jobs, "prune_removed_documents.searchdocument", f._threadService));
EXPECT_TRUE(containsJobAndExecutedBy(jobs, "move_buckets.searchdocument", f._threadService));
EXPECT_TRUE(containsJobAndExecutedBy(jobs, "sample_attribute_usage.searchdocument", f._threadService));
diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp
index 9f8f7fd9e4a..baf5819fdaf 100644
--- a/searchcore/src/tests/proton/server/feedstates_test.cpp
+++ b/searchcore/src/tests/proton/server/feedstates_test.cpp
@@ -52,7 +52,6 @@ MyFeedView::~MyFeedView() {}
struct MyReplayConfig : IReplayConfig {
virtual void replayConfig(SerialNum) override {}
- virtual void replayWipeHistory(SerialNum, fastos::TimeStamp) override {}
};
struct InstantExecutor : vespalib::Executor {
diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp
index 8b21ed40fc8..8fc1bd2d9ed 100644
--- a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp
+++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp
@@ -33,65 +33,49 @@ getConfig(int64_t generation)
}
-Schema::SP
-getSchema(int step)
-{
- Schema::SP schema(new Schema);
- schema->addIndexField(Schema::IndexField("foo1", DataType::STRING));
- if (step < 2) {
- schema->addIndexField(Schema::IndexField("foo2", DataType::STRING));
- }
- if (step < 1) {
- schema->addIndexField(Schema::IndexField("foo3", DataType::STRING));
- }
- return schema;
-}
-
TEST("require that configs can be stored and loaded") {
MemoryConfigStore config_store;
SerialNum serial(12);
- config_store.saveConfig(*getConfig(10), Schema(), serial);
+ config_store.saveConfig(*getConfig(10), serial);
DocumentDBConfig::SP config;
- Schema::SP history;
- config_store.loadConfig(*getConfig(14), serial, config, history);
+ config_store.loadConfig(*getConfig(14), serial, config);
ASSERT_TRUE(config.get());
- ASSERT_TRUE(history.get());
EXPECT_EQUAL(10, config->getGeneration());
}
TEST("require that best serial number is the most recent one") {
MemoryConfigStore config_store;
EXPECT_EQUAL(0u, config_store.getBestSerialNum());
- config_store.saveConfig(*getConfig(10), Schema(), 5);
+ config_store.saveConfig(*getConfig(10), 5);
EXPECT_EQUAL(5u, config_store.getBestSerialNum());
- config_store.saveConfig(*getConfig(10), Schema(), 2);
+ config_store.saveConfig(*getConfig(10), 2);
EXPECT_EQUAL(5u, config_store.getBestSerialNum());
}
TEST("require that oldest serial number is the first one or 0") {
MemoryConfigStore config_store;
EXPECT_EQUAL(0u, config_store.getOldestSerialNum());
- config_store.saveConfig(*getConfig(10), Schema(), 5);
+ config_store.saveConfig(*getConfig(10), 5);
EXPECT_EQUAL(5u, config_store.getOldestSerialNum());
- config_store.saveConfig(*getConfig(10), Schema(), 2);
+ config_store.saveConfig(*getConfig(10), 2);
EXPECT_EQUAL(2u, config_store.getOldestSerialNum());
}
TEST("require that existing serial numbers are valid") {
MemoryConfigStore config_store;
EXPECT_FALSE(config_store.hasValidSerial(5));
- config_store.saveConfig(*getConfig(10), Schema(), 5);
+ config_store.saveConfig(*getConfig(10), 5);
EXPECT_TRUE(config_store.hasValidSerial(5));
}
TEST("require that prev valid serial number is the last one before the arg") {
MemoryConfigStore config_store;
EXPECT_EQUAL(0u, config_store.getPrevValidSerial(10));
- config_store.saveConfig(*getConfig(10), Schema(), 5);
+ config_store.saveConfig(*getConfig(10), 5);
EXPECT_EQUAL(5u, config_store.getPrevValidSerial(10));
EXPECT_EQUAL(0u, config_store.getPrevValidSerial(5));
EXPECT_EQUAL(0u, config_store.getPrevValidSerial(4));
- config_store.saveConfig(*getConfig(10), Schema(), 2);
+ config_store.saveConfig(*getConfig(10), 2);
EXPECT_EQUAL(0u, config_store.getPrevValidSerial(1));
EXPECT_EQUAL(0u, config_store.getPrevValidSerial(2));
EXPECT_EQUAL(2u, config_store.getPrevValidSerial(4));
@@ -101,8 +85,8 @@ TEST("require that prev valid serial number is the last one before the arg") {
TEST("require that prune removes old configs") {
MemoryConfigStore config_store;
- config_store.saveConfig(*getConfig(10), Schema(), 5);
- config_store.saveConfig(*getConfig(10), Schema(), 6);
+ config_store.saveConfig(*getConfig(10), 5);
+ config_store.saveConfig(*getConfig(10), 6);
EXPECT_TRUE(config_store.hasValidSerial(5));
config_store.prune(5);
EXPECT_FALSE(config_store.hasValidSerial(5));
@@ -111,82 +95,12 @@ TEST("require that prune removes old configs") {
EXPECT_FALSE(config_store.hasValidSerial(6));
}
-TEST("require that wipe history clears previous history schema "
- "and adds new, identical entry for current serial num") {
- MemoryConfigStore config_store;
- Schema::SP history(new Schema);
- history->addIndexField(Schema::IndexField("foo", DataType::STRING));
- config_store.saveConfig(*getConfig(10), *history, 5);
- DocumentDBConfig::SP config;
- config_store.loadConfig(*getConfig(14), 5, config, history);
- EXPECT_EQUAL(1u, history->getNumIndexFields());
- config_store.saveWipeHistoryConfig(6, 0);
- EXPECT_TRUE(config_store.hasValidSerial(6));
- config_store.loadConfig(*getConfig(14), 5, config, history);
- EXPECT_EQUAL(1u, history->getNumIndexFields());
- config_store.loadConfig(*getConfig(14), 6, config, history);
- ASSERT_TRUE(config.get());
- ASSERT_TRUE(history.get());
- EXPECT_EQUAL(0u, history->getNumIndexFields());
-}
-
-
-TEST("require that wipe history clears only portions of history")
-{
- MemoryConfigStore config_store;
- Schema::SP schema(getSchema(0));
- Schema::SP history(new Schema);
- DocumentDBConfig::SP config(getConfig(5, schema));
- config_store.saveConfig(*config, *history, 5);
- Schema::SP oldSchema(schema);
- schema = getSchema(1);
- history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
- 100);
- config_store.saveConfig(*config, *history, 10);
- oldSchema = schema;
- schema = getSchema(2);
- history = SchemaUtil::makeHistorySchema(*schema, *oldSchema, *history,
- 200);
- config_store.saveConfig(*config, *history, 15);
- config_store.saveWipeHistoryConfig(20, 50);
- config_store.saveWipeHistoryConfig(25, 100);
- config_store.saveWipeHistoryConfig(30, 150);
- config_store.saveWipeHistoryConfig(35, 200);
- config_store.saveWipeHistoryConfig(40, 250);
- DocumentDBConfig::SP oldconfig(config);
- config_store.loadConfig(*oldconfig, 20, config, history);
- EXPECT_EQUAL(2u, history->getNumIndexFields());
- oldconfig = config;
- config_store.loadConfig(*oldconfig, 25, config, history);
- EXPECT_EQUAL(2u, history->getNumIndexFields());
- oldconfig = config;
- config_store.loadConfig(*oldconfig, 30, config, history);
- EXPECT_EQUAL(1u, history->getNumIndexFields());
- oldconfig = config;
- config_store.loadConfig(*oldconfig, 35, config, history);
- EXPECT_EQUAL(1u, history->getNumIndexFields());
- oldconfig = config;
- config_store.loadConfig(*oldconfig, 40, config, history);
- EXPECT_EQUAL(0u, history->getNumIndexFields());
-}
-
-TEST("require that wipe history does nothing if serial num exists") {
- MemoryConfigStore config_store;
- Schema::SP history(new Schema);
- history->addIndexField(Schema::IndexField("foo", DataType::STRING));
- config_store.saveConfig(*getConfig(10), *history, 5);
- DocumentDBConfig::SP config;
- config_store.saveWipeHistoryConfig(5, 0);
- config_store.loadConfig(*getConfig(14), 5, config, history);
- EXPECT_EQUAL(1u, history->getNumIndexFields());
-}
-
TEST("require that MemoryConfigStores preserves state of "
"MemoryConfigStore between instantiations") {
MemoryConfigStores config_stores;
const std::string name("foo");
ConfigStore::UP config_store = config_stores.getConfigStore(name);
- config_store->saveConfig(*getConfig(10), Schema(), 5);
+ config_store->saveConfig(*getConfig(10), 5);
EXPECT_TRUE(config_store->hasValidSerial(5));
config_store.reset();
config_store = config_stores.getConfigStore(name);
diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
index 2bccad4f824..18c000e5f4e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
@@ -97,7 +97,6 @@ vespa_add_library(searchcore_server STATIC
transactionlogmanagerbase.cpp
updatedonecontext.cpp
visibilityhandler.cpp
- wipe_old_removed_fields_job.cpp
DEPENDS
searchcore_attribute
searchcore_bucketdb
diff --git a/searchcore/src/vespa/searchcore/proton/server/configstore.h b/searchcore/src/vespa/searchcore/proton/server/configstore.h
index 38d0dcfe7bd..1c76cf2d4da 100644
--- a/searchcore/src/vespa/searchcore/proton/server/configstore.h
+++ b/searchcore/src/vespa/searchcore/proton/server/configstore.h
@@ -21,15 +21,11 @@ struct ConfigStore : FeedConfigStore {
* @param serialNum the serial number of the config snapshot to load.
* @param loadedSnapshot the shared pointer in which to store the
* resulting config snapshot.
- * @param historySchema the shared pointer in which to store the
- * resulting history schema.
*/
virtual void loadConfig(const DocumentDBConfig &currentSnapshot,
SerialNum serialNum,
- DocumentDBConfig::SP &loadedSnapshot,
- search::index::Schema::SP &historySchema) = 0;
+ DocumentDBConfig::SP &loadedSnapshot) = 0;
virtual void saveConfig(const DocumentDBConfig &snapshot,
- const search::index::Schema &historySchema,
SerialNum serialNum) = 0;
virtual void removeInvalid() = 0;
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp
index f3911c5e122..04e2557e2ca 100644
--- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp
@@ -99,8 +99,6 @@ DocumentDBMaintenanceConfig::
DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig &
pruneRemovedDocuments,
const DocumentDBHeartBeatConfig &heartBeat,
- const DocumentDBWipeOldRemovedFieldsConfig &
- wipeOldRemovedFields,
double groupingSessionPruneInterval,
fastos::TimeStamp visibilityDelay,
const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction,
@@ -109,7 +107,6 @@ DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig &
double resourceLimitFactor)
: _pruneRemovedDocuments(pruneRemovedDocuments),
_heartBeat(heartBeat),
- _wipeOldRemovedFields(wipeOldRemovedFields),
_sessionCachePruneInterval(groupingSessionPruneInterval),
_visibilityDelay(visibilityDelay),
_lidSpaceCompaction(lidSpaceCompaction),
@@ -125,7 +122,6 @@ operator==(const DocumentDBMaintenanceConfig &rhs) const
{
return _pruneRemovedDocuments == rhs._pruneRemovedDocuments &&
_heartBeat == rhs._heartBeat &&
- _wipeOldRemovedFields == rhs._wipeOldRemovedFields &&
_sessionCachePruneInterval == rhs._sessionCachePruneInterval &&
_visibilityDelay == rhs._visibilityDelay &&
_lidSpaceCompaction == rhs._lidSpaceCompaction &&
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h
index 199a7716dd1..583a676a01d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h
@@ -23,7 +23,6 @@ public:
};
typedef DocumentDBPruneConfig DocumentDBPruneRemovedDocumentsConfig;
-typedef DocumentDBPruneConfig DocumentDBWipeOldRemovedFieldsConfig;
class DocumentDBHeartBeatConfig
{
@@ -72,7 +71,6 @@ public:
private:
DocumentDBPruneRemovedDocumentsConfig _pruneRemovedDocuments;
DocumentDBHeartBeatConfig _heartBeat;
- DocumentDBWipeOldRemovedFieldsConfig _wipeOldRemovedFields;
double _sessionCachePruneInterval;
fastos::TimeStamp _visibilityDelay;
DocumentDBLidSpaceCompactionConfig _lidSpaceCompaction;
@@ -85,7 +83,6 @@ public:
DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig &pruneRemovedDocuments,
const DocumentDBHeartBeatConfig &heartBeat,
- const DocumentDBWipeOldRemovedFieldsConfig &wipeOldRemovedFields,
double sessionCachePruneInterval,
fastos::TimeStamp visibilityDelay,
const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction,
@@ -108,12 +105,6 @@ public:
return _heartBeat;
}
- const DocumentDBWipeOldRemovedFieldsConfig &
- getWipeOldRemovedFieldsConfig() const
- {
- return _wipeOldRemovedFields;
- }
-
double
getSessionCachePruneInterval() const
{
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index c639b36b592..631e9d0ff18 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -102,7 +102,6 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
IFeedHandlerOwner(),
IDocumentSubDBOwner(),
IClusterStateChangedHandler(),
- IWipeOldRemovedFieldsHandler(),
search::transactionlog::SyncProxy(),
_docTypeName(docTypeName),
_baseDir(baseDir + "/" + _docTypeName.toString()),
@@ -147,7 +146,6 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
_writeFilter,
*this,
tlsDirectWriter),
- _historySchema(),
_subDBs(*this,
*this,
_feedHandler,
@@ -185,14 +183,13 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
assert(configSerial > 0);
DocumentDBConfig::SP loaded_config;
_config_store->loadConfig(*configSnapshot, configSerial,
- loaded_config, _historySchema);
+ loaded_config);
// Grab relevant parts from pending config
loaded_config = DocumentDBConfigScout::scout(loaded_config,
*_pendingConfigSnapshot.get());
// Ignore configs that are not relevant during replay of transaction log
loaded_config = DocumentDBConfig::makeReplayConfig(loaded_config);
- reconfigureSchema(*loaded_config, *loaded_config);
_initConfigSnapshot = loaded_config;
_initConfigSerialNum = configSerial;
// Forward changes of cluster state to feed view via us
@@ -438,9 +435,6 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot,
cmpres = _activeConfigSnapshot->compare(*configSnapshot);
}
const ReconfigParams params(cmpres);
- if (params.shouldSchemaChange()) {
- reconfigureSchema(*configSnapshot, *_activeConfigSnapshot);
- }
// Save config via config manager if replay is done.
bool equalReplayConfig =
*DocumentDBConfig::makeReplayConfig(configSnapshot) ==
@@ -449,7 +443,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot,
if (!equalReplayConfig && tlsReplayDone) {
sync(_feedHandler.getSerialNum());
serialNum = _feedHandler.incSerialNum();
- _config_store->saveConfig(*configSnapshot, *_historySchema, serialNum);
+ _config_store->saveConfig(*configSnapshot, serialNum);
// save entry in transaction log
NewConfigOperation op(serialNum, *_config_store);
_feedHandler.storeOperation(op);
@@ -502,19 +496,6 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot,
}
-void
-DocumentDB::reconfigureSchema(const DocumentDBConfig &configSnapshot,
- const DocumentDBConfig &oldConfigSnapshot)
-{
- // Called by CTOR and executor thread
- const Schema &newSchema = *configSnapshot.getSchemaSP();
- const Schema &oldSchema = *oldConfigSnapshot.getSchemaSP();
- Schema::SP oldHistory = _historySchema;
- _historySchema =
- SchemaUtil::makeHistorySchema(newSchema, oldSchema, *oldHistory);
-}
-
-
namespace {
void
doNothing(IFeedView::SP)
@@ -663,7 +644,7 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot)
LOG(warning, "DocumentDB(%s): saveInitialConfig() failed pruning due to '%s'",
_docTypeName.toString().c_str(), e.what());
}
- _config_store->saveConfig(configSnapshot, Schema(), confSerial);
+ _config_store->saveConfig(configSnapshot, confSerial);
}
@@ -932,40 +913,6 @@ void
DocumentDB::performWipeHistory()
{
// Called by executor thread
- if (_historySchema->empty())
- return;
- if (_feedHandler.getTransactionLogReplayDone()) {
- sync(_feedHandler.getSerialNum()); // Sync before wiping history
- DocumentDBConfig::SP configSnapshot = getActiveConfig();
- SerialNum wipeSerial = _feedHandler.incSerialNum();
- Schema::UP newHistory(new Schema);
- writeWipeHistoryTransactionLogEntry(wipeSerial, 0,
- *configSnapshot, *newHistory);
- internalWipeHistory(wipeSerial, std::move(newHistory));
- }
-}
-
-
-void DocumentDB::writeWipeHistoryTransactionLogEntry(
- SerialNum wipeSerial, fastos::TimeStamp wipeTimeLimit,
- const DocumentDBConfig &configSnapshot,
- const Schema &newHistorySchema) {
- // Caller must have synced transaction log
- _config_store->saveConfig(configSnapshot, newHistorySchema, wipeSerial);
- // save entry in transaction log
- WipeHistoryOperation op(wipeSerial, wipeTimeLimit);
- _feedHandler.storeOperation(op);
- sync(op.getSerialNum());
-}
-
-
-void
-DocumentDB::internalWipeHistory(SerialNum wipeSerial,
- Schema::UP newHistorySchema)
-{
- // Called by executor thread
- _subDBs.wipeHistory(wipeSerial);
- _historySchema.reset(newHistorySchema.release());
}
@@ -980,10 +927,9 @@ DocumentDB::replayConfig(search::SerialNum serialNum)
_docTypeName.toString().c_str(), serialNum);
return;
}
- // Load historyschema before applyConfig to preserve the history
- // field timestamps.
+ // Load config to replay
_config_store->loadConfig(*configSnapshot, serialNum,
- configSnapshot, _historySchema);
+ configSnapshot);
// Grab relevant parts from pending config
configSnapshot = DocumentDBConfigScout::scout(configSnapshot,
*_pendingConfigSnapshot.get());
@@ -996,36 +942,6 @@ DocumentDB::replayConfig(search::SerialNum serialNum)
}
void
-DocumentDB::replayWipeHistory(search::SerialNum serialNum,
- fastos::TimeStamp wipeTimeLimit)
-{
- // Called by executor thread
- DocumentDBConfig::SP configSnapshot = getActiveConfig();
- if (configSnapshot.get() == NULL) {
- LOG(warning,
- "DocumentDB(%s): Missing old config when replaying wipe history, serialNum=%" PRIu64,
- _docTypeName.toString().c_str(),
- serialNum);
- return;
- }
- Schema::UP wipeSchemaOwner;
- Schema *wipeSchema;
- Schema::UP newHistory;
- if (wipeTimeLimit) {
- wipeSchemaOwner = _historySchema->getOldFields(wipeTimeLimit);
- wipeSchema = wipeSchemaOwner.get();
- newHistory = Schema::set_difference(*_historySchema, *wipeSchema);
- } else { // wipeTimeLimit == 0 means old style wipeHistory.
- wipeSchema = _historySchema.get();
- newHistory.reset(new Schema);
- }
- LOG(info, "DocumentDB(%s): Replayed history wipe with serialNum=%" PRIu64,
- _docTypeName.toString().c_str(), serialNum);
- internalWipeHistory(serialNum, std::move(newHistory));
-}
-
-
-void
DocumentDB::listSchema(std::vector<vespalib::string> &fieldNames,
std::vector<vespalib::string> &fieldDataTypes,
std::vector<vespalib::string> &fieldCollectionTypes,
@@ -1087,7 +1003,6 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config)
config,
_feedHandler, // IHeartBeatHandler
*_sessionManager, // ISessionCachePruner
- *this, // IWipeOldRemovedFieldsHandler
_lidSpaceCompactionHandlers,
_feedHandler, // IOperationStorer
_maintenanceController, // IFrozenBucketHandler
@@ -1198,51 +1113,6 @@ DocumentDB::notifyAllBucketsChanged()
}
-namespace {
-
-vespalib::string
-getSchemaFieldsList(const Schema &schema)
-{
- vespalib::asciistream oss;
- for (uint32_t i = 0; i < schema.getNumIndexFields(); ++i) {
- if (i > 0) oss << ",";
- oss << schema.getIndexField(i).getName();
- }
- for (uint32_t i = 0; i < schema.getNumAttributeFields(); ++i) {
- if (!oss.str().empty()) oss << ",";
- oss << schema.getAttributeField(i).getName();
- }
- return oss.str();
-}
-
-}
-
-void
-DocumentDB::wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit)
-{
- // Called by executor thread
-
- if (_historySchema->empty())
- return;
- DocumentDBConfig::SP configSnapshot = getActiveConfig();
-
- Schema::UP wipeSchema = _historySchema->getOldFields(wipeTimeLimit);
- Schema::UP newHistorySchema =
- Schema::set_difference(*_historySchema, *wipeSchema);
-
- sync(_feedHandler.getSerialNum()); // Sync before wiping history
- SerialNum wipeSerial = _feedHandler.incSerialNum();
- writeWipeHistoryTransactionLogEntry(wipeSerial, wipeTimeLimit,
- *configSnapshot, *newHistorySchema);
- internalWipeHistory(wipeSerial, std::move(newHistorySchema));
-
- LOG(debug, "DocumentDB(%s): Done wipeOldRemovedFields: wipe(%s), history(%s) timeLimit(%" PRIu64 ")",
- _docTypeName.toString().c_str(),
- getSchemaFieldsList(*wipeSchema).c_str(),
- getSchemaFieldsList(*_historySchema).c_str(),
- static_cast<uint64_t>(wipeTimeLimit.sec()));
-}
-
searchcorespi::IIndexManagerFactory::SP
DocumentDB::getIndexManagerFactory(const vespalib::stringref &name) const
{
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 3705c8d786d..fea0439aede 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -13,7 +13,6 @@
#include "i_lid_space_compaction_handler.h"
#include "ifeedview.h"
#include "ireplayconfig.h"
-#include "iwipeoldremovedfieldshandler.h"
#include "maintenancecontroller.h"
#include "i_document_db_config_owner.h"
#include "executorthreadingservice.h"
@@ -65,7 +64,6 @@ class DocumentDB : public IDocumentDBConfigOwner,
public IFeedHandlerOwner,
public IDocumentSubDBOwner,
public IClusterStateChangedHandler,
- public IWipeOldRemovedFieldsHandler,
public search::transactionlog::SyncProxy
{
private:
@@ -133,11 +131,6 @@ private:
AttributeUsageFilter _writeFilter;
FeedHandler _feedHandler;
- // Members only accessed by executor thread
- // (+ ctor and after executor stops)
- Schema::SP _historySchema; // Removed fields
- // End members only accessed by executor thread.
-
DocumentSubDBCollection _subDBs;
MaintenanceController _maintenanceController;
VisibilityHandler _visibility;
@@ -172,10 +165,6 @@ private:
*/
void resumeSaveConfig(void);
- void
- reconfigureSchema(const DocumentDBConfig &configSnapshot,
- const DocumentDBConfig &oldConfigSnapshot);
-
void setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum serialNum);
/**
@@ -227,10 +216,6 @@ private:
virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override;
void notifyAllBucketsChanged();
- /**
- * Implements IWipeOldRemovedFieldsHandler
- */
- virtual void wipeOldRemovedFields(TimeStamp wipeTimeLimit) override;
void updateLegacyMetrics(LegacyDocumentDBMetrics &metrics);
void updateMetrics(DocumentDBTaggedMetrics &metrics);
void updateMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics);
@@ -430,8 +415,6 @@ public:
*/
virtual void replayConfig(SerialNum serialNum) override;
- virtual void replayWipeHistory(SerialNum serialNum, TimeStamp wipeTimeLimit) override;
-
const DocTypeName & getDocTypeName(void) const { return _docTypeName; }
void
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index a258a282f81..0006007caed 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -120,9 +120,6 @@ buildMaintenanceConfig(const BootstrapConfig::SP &bootstrapConfig,
pruneRemovedDocumentsInterval,
pruneRemovedDocumentsAge),
DocumentDBHeartBeatConfig(),
- DocumentDBWipeOldRemovedFieldsConfig(
- proton.wipeoldremovedfieldsinterval,
- proton.wipeoldremovedfieldsage),
proton.grouping.sessionmanager.pruning.interval,
visibilityDelay,
DocumentDBLidSpaceCompactionConfig(
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h
index bfc653ac966..5320793d9ab 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h
+++ b/searchcore/src/vespa/searchcore/proton/server/feedconfigstore.h
@@ -9,8 +9,6 @@ namespace proton {
struct FeedConfigStore : NewConfigOperation::IStreamHandler {
virtual ~FeedConfigStore() {}
- virtual void saveWipeHistoryConfig(search::SerialNum,
- fastos::TimeStamp wipeTimeLimit) = 0;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
index 2e79db1e599..516dd8f74cc 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp
@@ -91,11 +91,7 @@ public:
virtual void replay(const NewConfigOperation &op) override {
_replay_config.replayConfig(op.getSerialNum());
}
- virtual void replay(const WipeHistoryOperation &op) override {
- _config_store.saveWipeHistoryConfig(op.getSerialNum(),
- op.getWipeTimeLimit());
- _replay_config.replayWipeHistory(op.getSerialNum(),
- op.getWipeTimeLimit());
+ virtual void replay(const WipeHistoryOperation &) override {
}
virtual void replay(const DeleteBucketOperation &op) override {
_feed_view_ptr->handleDeleteBucket(op);
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
index 07d8ffabb62..fe8f3210a51 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
@@ -298,7 +298,6 @@ FileConfigManager::getOldestSerialNum(void) const
void
FileConfigManager::saveConfig(const DocumentDBConfig &snapshot,
- const search::index::Schema &historySchema,
SerialNum serialNum)
{
if (getBestSerialNum() >= serialNum) {
@@ -326,10 +325,6 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot,
assert(saveSchemaRes);
(void) saveSchemaRes;
- bool saveHistorySchemaRes = historySchema.saveToFile(snapDir + "/historyschema.txt");
- assert(saveHistorySchemaRes);
- (void) saveHistorySchemaRes;
-
writeExtraConfigs(snapDir, snapshot);
_info.validateSnapshot(serialNum);
@@ -361,8 +356,7 @@ void addEmptyFile(vespalib::string snapDir, vespalib::string fileName)
void
FileConfigManager::loadConfig(const DocumentDBConfig &currentSnapshot,
search::SerialNum serialNum,
- DocumentDBConfig::SP &loadedSnapshot,
- search::index::Schema::SP &historySchema)
+ DocumentDBConfig::SP &loadedSnapshot)
{
vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
@@ -404,15 +398,9 @@ FileConfigManager::loadConfig(const DocumentDBConfig &currentSnapshot,
dbc.forwardConfig(bootstrap);
dbc.nextGeneration(0);
- Schema::UP newHistorySchema(new Schema);
- bool loadHistorySchemaRes = newHistorySchema->loadFromFile(snapDir + "/historyschema.txt");
- assert(loadHistorySchemaRes);
- (void) loadHistorySchemaRes;
-
loadedSnapshot = dbc.getConfig();
loadedSnapshot->setConfigId(_configId);
loadedSnapshot->setExtraConfigs(readExtraConfigs(snapDir));
- historySchema.reset(newHistorySchema.release());
}
@@ -498,65 +486,6 @@ FileConfigManager::getPrevValidSerial(SerialNum serialNum) const
void
-FileConfigManager::saveWipeHistoryConfig(SerialNum serialNum,
- fastos::TimeStamp wipeTimeLimit)
-{
- vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
- vespalib::string snapDir(_baseDir + "/" + snapDirBaseName);
-
- if (hasValidSerial(serialNum))
- return; // config already saved.
-
- SerialNum prevSerialNum = getPrevValidSerial(serialNum);
-
- assert(prevSerialNum > 0);
-
- Snapshot snap(false, serialNum, snapDirBaseName);
- _info.addSnapshot(snap);
-
- bool saveInvalidSnap = _info.save();
- assert(saveInvalidSnap);
- (void) saveInvalidSnap;
-
- vespalib::mkdir(snapDir, false);
-
- vespalib::string prevSnapDirBaseName(makeSnapDirBaseName(prevSerialNum));
- vespalib::string prevSnapDir(_baseDir + "/" + prevSnapDirBaseName);
-
- std::vector<vespalib::string> configs = getFileList(prevSnapDir);
- for (const auto &config : configs) {
- if (config == "historyschema.txt") {
- Schema::UP historySchema(new Schema);
- if (wipeTimeLimit != 0) {
- Schema oldHistorySchema;
- bool loadOldHistorySchemaRes =
- oldHistorySchema.loadFromFile(prevSnapDir +
- "/historyschema.txt");
- assert(loadOldHistorySchemaRes);
- (void) loadOldHistorySchemaRes;
- Schema::UP wipeSchema;
- wipeSchema = oldHistorySchema.getOldFields(wipeTimeLimit);
- historySchema = Schema::set_difference(oldHistorySchema,
- *wipeSchema);
- }
- bool saveHistorySchemaRes =
- historySchema->saveToFile(snapDir + "/historyschema.txt");
- assert(saveHistorySchemaRes);
- (void) saveHistorySchemaRes;
- continue;
- }
- ConfigFile file(config,
- prevSnapDir + "/" + config);
- file.save(snapDir);
- }
- _info.validateSnapshot(serialNum);
- bool saveValidSnap = _info.save();
- assert(saveValidSnap);
- (void) saveValidSnap;
-}
-
-
-void
FileConfigManager::serializeConfig(SerialNum serialNum, nbostream &stream)
{
vespalib::string snapDirBaseName(makeSnapDirBaseName(serialNum));
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h
index 3a108e866b3..eb8b1a8c690 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h
@@ -40,7 +40,6 @@ public:
virtual SerialNum getOldestSerialNum() const override;
virtual void saveConfig(const DocumentDBConfig &snapshot,
- const search::index::Schema &historySchema,
SerialNum serialNum) override;
/**
@@ -53,13 +52,10 @@ public:
* @param serialNum the serial number of the config snapshot to load.
* @param loadedSnapshot the shared pointer in which to store the
* resulting config snapshot.
- * @param historySchema the shared pointer in which to store the
- * resulting history schema.
*/
virtual void loadConfig(const DocumentDBConfig &currentSnapshot,
SerialNum serialNum,
- DocumentDBConfig::SP &loadedSnapshot,
- search::index::Schema::SP &historySchema) override;
+ DocumentDBConfig::SP &loadedSnapshot) override;
virtual void removeInvalid() override;
virtual void prune(SerialNum serialNum) override;
@@ -68,14 +64,6 @@ public:
virtual SerialNum getPrevValidSerial(SerialNum serialNum) const override;
/**
- * Clone config except for history schema.
- * Used when wiping history.
- */
- virtual void saveWipeHistoryConfig(SerialNum serialNum,
- fastos::TimeStamp wipeTimeLimit) override;
-
-
- /**
* Serialize config files.
*
* Used for serializing config into transaction log.
diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h
index 866eeba4bea..6da62888208 100644
--- a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h
@@ -15,10 +15,6 @@ public:
virtual void
replayConfig(search::SerialNum serialNum) = 0;
-
- virtual void
- replayWipeHistory(search::SerialNum serialNum,
- fastos::TimeStamp wipeTimeLimit) = 0;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h b/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h
deleted file mode 100644
index 95f70ca308c..00000000000
--- a/searchcore/src/vespa/searchcore/proton/server/iwipeoldremovedfieldshandler.h
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-namespace proton {
-
-class IWipeOldRemovedFieldsHandler
-{
-public:
- virtual void wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit) = 0;
-
- virtual ~IWipeOldRemovedFieldsHandler() {}
-};
-
-} // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
index 9217913e6a6..a1b23ea65c5 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp
@@ -9,7 +9,6 @@
#include "prune_session_cache_job.h"
#include "pruneremoveddocumentsjob.h"
#include "sample_attribute_usage_job.h"
-#include "wipe_old_removed_fields_job.h"
using fastos::ClockSystem;
using fastos::TimeStamp;
@@ -81,7 +80,6 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller,
const DocumentDBMaintenanceConfig &config,
IHeartBeatHandler &hbHandler,
matching::ISessionCachePruner &scPruner,
- IWipeOldRemovedFieldsHandler &worfHandler,
const ILidSpaceCompactionHandler::Vector &lscHandlers,
IOperationStorer &opStorer,
IFrozenBucketHandler &fbHandler,
@@ -104,8 +102,6 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller,
if (config.getVisibilityDelay() > 0) {
controller.registerJobInMasterThread(MUP(new DocumentDBCommitJob(commit, config.getVisibilityDelay())));
}
- controller.registerJobInMasterThread(
- MUP(new WipeOldRemovedFieldsJob(worfHandler, config.getWipeOldRemovedFieldsConfig())));
const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB());
MUP pruneRDjob(new PruneRemovedDocumentsJob(config.getPruneRemovedDocumentsConfig(), *mRemSubDB._metaStore,
mRemSubDB._subDbId, docTypeName, prdHandler, fbHandler));
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
index dfad0947c8c..eff3442c49e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h
@@ -6,7 +6,6 @@
#include "i_lid_space_compaction_handler.h"
#include "i_operation_storer.h"
#include "iheartbeathandler.h"
-#include "iwipeoldremovedfieldshandler.h"
#include "icommitable.h"
#include <vespa/searchcore/proton/matching/isessioncachepruner.h>
#include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
@@ -34,7 +33,6 @@ struct MaintenanceJobsInjector
const DocumentDBMaintenanceConfig &config,
IHeartBeatHandler &hbHandler,
matching::ISessionCachePruner &scPruner,
- IWipeOldRemovedFieldsHandler &worfHandler,
const ILidSpaceCompactionHandler::Vector &lscHandlers,
IOperationStorer &opStorer,
IFrozenBucketHandler &fbHandler,
diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp
index b070e2d8fc2..c687675e0d9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp
@@ -33,21 +33,17 @@ MemoryConfigStore::getPrevValidSerial(SerialNum serial) const {
}
void
MemoryConfigStore::saveConfig(const DocumentDBConfig &config,
- const Schema &history,
SerialNum serial)
{
_maps->configs[serial].reset(new DocumentDBConfig(config));
- _maps->histories[serial].reset(new Schema(history));
_maps->_valid.insert(serial);
}
void
MemoryConfigStore::loadConfig(const DocumentDBConfig &, SerialNum serial,
- DocumentDBConfig::SP &loaded_config,
- Schema::SP &history_schema)
+ DocumentDBConfig::SP &loaded_config)
{
assert(hasValidSerial(serial));
loaded_config = _maps->configs[serial];
- history_schema = _maps->histories[serial];
}
void
MemoryConfigStore::removeInvalid()
@@ -60,44 +56,16 @@ MemoryConfigStore::removeInvalid()
}
++it;
}
- for (auto it = _maps->histories.begin();
- it != _maps->histories.end();) {
- if (!hasValidSerial(it->first)) {
- it = _maps->histories.erase(it);
- continue;
- }
- ++it;
- }
}
+
void
MemoryConfigStore::prune(SerialNum serial) {
_maps->configs.erase(_maps->configs.begin(),
_maps->configs.upper_bound(serial));
- _maps->histories.erase(_maps->histories.begin(),
- _maps->histories.upper_bound(serial));
_maps->_valid.erase(_maps->_valid.begin(),
_maps->_valid.upper_bound(serial));
}
-void
-MemoryConfigStore::saveWipeHistoryConfig(SerialNum serial,
- fastos::TimeStamp wipeTimeLimit)
-{
- if (hasValidSerial(serial)) {
- return;
- }
- SerialNum prev = getPrevValidSerial(serial);
- Schema::SP schema(new Schema);
- if (wipeTimeLimit != 0) {
- Schema::SP oldHistorySchema(_maps->histories[prev]);
- Schema::UP wipeSchema;
- wipeSchema = oldHistorySchema->getOldFields(wipeTimeLimit);
- schema.reset(Schema::set_difference(*oldHistorySchema, *wipeSchema).release());
-
- }
- _maps->histories[serial] = schema;
- _maps->configs[serial] = _maps->configs[prev];
- _maps->_valid.insert(serial);
-}
+
void
MemoryConfigStore::serializeConfig(SerialNum, vespalib::nbostream &) {
LOG(info, "Serialization of config not implemented.");
diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h
index 57411c81e61..24d6e96106d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h
+++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h
@@ -12,7 +12,6 @@ namespace proton {
struct ConfigMaps {
typedef std::shared_ptr<ConfigMaps> SP;
std::map<search::SerialNum, DocumentDBConfig::SP> configs;
- std::map<search::SerialNum, search::index::Schema::SP> histories;
std::set<search::SerialNum> _valid;
};
@@ -32,14 +31,11 @@ public:
virtual bool hasValidSerial(SerialNum serial) const override;
virtual SerialNum getPrevValidSerial(SerialNum serial) const override;
virtual void saveConfig(const DocumentDBConfig &config,
- const Schema &history,
SerialNum serial) override;
virtual void loadConfig(const DocumentDBConfig &, SerialNum serial,
- DocumentDBConfig::SP &loaded_config,
- Schema::SP &history_schema) override;
+ DocumentDBConfig::SP &loaded_config) override;
virtual void removeInvalid() override;
void prune(SerialNum serial) override;
- virtual void saveWipeHistoryConfig(SerialNum serial, fastos::TimeStamp wipeTimeLimit) override;
virtual void serializeConfig(SerialNum, vespalib::nbostream &) override;
virtual void deserializeConfig(SerialNum, vespalib::nbostream &) override;
virtual void setProtonConfig(const ProtonConfigSP &) override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp
deleted file mode 100644
index dff3ff57375..00000000000
--- a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/fastos/fastos.h>
-#include "wipe_old_removed_fields_job.h"
-#include <vespa/fastos/timestamp.h>
-
-using fastos::ClockSystem;
-using fastos::TimeStamp;
-
-namespace proton {
-
-WipeOldRemovedFieldsJob::WipeOldRemovedFieldsJob(IWipeOldRemovedFieldsHandler &handler,
- const DocumentDBWipeOldRemovedFieldsConfig &config)
- : IMaintenanceJob("wipe_old_removed_fields", config.getInterval(), config.getInterval()),
- _handler(handler),
- _ageLimitSeconds(config.getAge())
-{
-}
-
-bool
-WipeOldRemovedFieldsJob::run()
-{
- TimeStamp wipeTimeLimit = TimeStamp(ClockSystem::now()) -
- TimeStamp(int64_t(_ageLimitSeconds) * TimeStamp::SEC);
- _handler.wipeOldRemovedFields(wipeTimeLimit);
- return true;
-}
-
-} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h b/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h
deleted file mode 100644
index eb8ec4a7649..00000000000
--- a/searchcore/src/vespa/searchcore/proton/server/wipe_old_removed_fields_job.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "document_db_maintenance_config.h"
-#include "i_maintenance_job.h"
-#include "iwipeoldremovedfieldshandler.h"
-
-namespace proton {
-
-/**
- * Job that regularly wipes old removed fields from a document database.
- */
-class WipeOldRemovedFieldsJob : public IMaintenanceJob
-{
-private:
- IWipeOldRemovedFieldsHandler &_handler;
- const double _ageLimitSeconds;
-
-public:
- WipeOldRemovedFieldsJob(IWipeOldRemovedFieldsHandler &handler,
- const DocumentDBWipeOldRemovedFieldsConfig &config);
-
- // Implements IMaintenanceJob
- virtual bool run() override;
-};
-
-} // namespace proton
-