Wipe removed fields when setting up index manager.

11 files changed, 25 insertions, 2 deletions

diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index 7d7c3604feb..0b4d101a6f3 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -188,7 +188,7 @@ Fixture::initViewSet(ViewSet &views)
 {
     Matchers::SP matchers(new Matchers(_clock, _queryLimiter, _constantValueRepo));
     IndexManager::SP indexMgr(new IndexManager(BASE_DIR, searchcorespi::index::WarmupConfig(),
-                                      2, 0, Schema(), views._reconfigurer,
+                                      2, 0, Schema(), 1, views._reconfigurer,
                                       views._writeService, _summaryExecutor, TuneFileIndexManager(),
                                       TuneFileAttributes(), views._fileHeaderContext));
     AttributeManager::SP attrMgr(new AttributeManager(BASE_DIR,
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp
index 620b4cb80bb..cf26b05bf7d 100644
--- a/searchcore/src/tests/proton/index/indexmanager_test.cpp
+++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp
@@ -176,7 +176,7 @@ Document::UP Fixture::addDocument(uint32_t id) {
 void Fixture::resetIndexManager() {
     _index_manager.reset(0);
     _index_manager.reset(
-            new IndexManager(index_dir, searchcorespi::index::WarmupConfig(), 2, 0, getSchema(),
+            new IndexManager(index_dir, searchcorespi::index::WarmupConfig(), 2, 0, getSchema(), 1,
                              _reconfigurer, _writeService, _writeService.getMasterExecutor(),
                              TuneFileIndexManager(), TuneFileAttributes(),
                              _fileHeaderContext));
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
index e7bd37d5f1f..4f882660855 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
@@ -15,6 +15,7 @@ IndexManagerInitializer(const vespalib::string &baseDir,
                         size_t maxFlushed,
                         size_t cacheSize,
                         const search::index::Schema &schema,
+                        search::SerialNum serialNum,
                         searchcorespi::IIndexManager::Reconfigurer & reconfigurer,
                         searchcorespi::index::IThreadingService & threadingService,
                         vespalib::ThreadExecutor & warmupExecutor,
@@ -27,6 +28,7 @@ IndexManagerInitializer(const vespalib::string &baseDir,
       _maxFlushed(maxFlushed),
       _cacheSize(cacheSize),
       _schema(schema),
+      _serialNum(serialNum),
       _reconfigurer(reconfigurer),
       _threadingService(threadingService),
       _warmupExecutor(warmupExecutor),
@@ -50,6 +52,7 @@ IndexManagerInitializer::run()
                      _maxFlushed,
                      _cacheSize,
                      _schema,
+                     _serialNum,
                      _reconfigurer,
                      _threadingService,
                      _warmupExecutor,
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
index 1bf6a13ce49..e383e9a29ee 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
@@ -20,6 +20,7 @@ class IndexManagerInitializer :  public initializer::InitializerTask
     size_t                                      _maxFlushed;
     size_t                                      _cacheSize;
     const search::index::Schema                 _schema;
+    search::SerialNum                           _serialNum;
     searchcorespi::IIndexManager::Reconfigurer &_reconfigurer;
     searchcorespi::index::IThreadingService    &_threadingService;
     vespalib::ThreadExecutor                   &_warmupExecutor;
@@ -34,6 +35,7 @@ public:
                             size_t maxFlushed,
                             size_t cacheSize,
                             const search::index::Schema &schema,
+                            search::SerialNum serialNum,
                             searchcorespi::IIndexManager::Reconfigurer & reconfigurer,
                             searchcorespi::index::IThreadingService & threadingService,
                             vespalib::ThreadExecutor & warmupExecutor,
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
index 9e754432251..5ba6e11331c 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
@@ -86,6 +86,7 @@ IndexManager::IndexManager(const vespalib::string &baseDir,
                            const size_t maxFlushed,
                            const size_t cacheSize,
                            const Schema &schema,
+                           SerialNum serialNum,
                            Reconfigurer &reconfigurer,
                            IThreadingService &threadingService,
                            vespalib::ThreadExecutor & warmupExecutor,
@@ -98,6 +99,7 @@ IndexManager::IndexManager(const vespalib::string &baseDir,
                                       warmup,
                                       maxFlushed,
                                       schema,
+                                      serialNum,
                                       tuneFileAttributes),
                 IndexMaintainerContext(threadingService,
                                        reconfigurer,
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
index 45e7b30b8f1..cd4631d234b 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
@@ -58,6 +58,7 @@ public:
                  size_t maxFlushed,
                  size_t cacheSize,
                  const Schema &schema,
+                 SerialNum serialNum,
                  Reconfigurer &reconfigurer,
                  searchcorespi::index::IThreadingService &threadingService,
                  vespalib::ThreadExecutor & warmupExecutor,
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index 2bba58f0522..223633c0e82 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -86,6 +86,7 @@ SearchableDocSubDB::getNewestFlushedSerial()
 initializer::InitializerTask::SP
 SearchableDocSubDB::
 createIndexManagerInitializer(const DocumentDBConfig &configSnapshot,
+                              SerialNum configSerialNum,
                               const ProtonConfig::Index &indexCfg,
                               std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager) const
 {
@@ -98,6 +99,7 @@ createIndexManagerInitializer(const DocumentDBConfig &configSnapshot,
          indexCfg.maxflushed,
          indexCfg.cache.size,
          *schema,
+         configSerialNum,
          const_cast<SearchableDocSubDB &>(*this),
          _writeService,
          _warmupExecutor,
@@ -127,6 +129,7 @@ createInitializer(const DocumentDBConfig &configSnapshot,
                                             protonSummaryCfg,
                                             indexCfg);
     auto indexTask = createIndexManagerInitializer(configSnapshot,
+                                                   configSerialNum,
                                                    indexCfg,
                                                    result->writableResult().
                                                    writableIndexManager());
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index 23f2ac2a8ac..d027db75cbb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -90,6 +90,7 @@ private:
     // Note: lifetime of indexManager must be handled by caller.
     std::shared_ptr<initializer::InitializerTask>
     createIndexManagerInitializer(const DocumentDBConfig &configSnapshot,
+                                  SerialNum configSerialNum,
                                   const vespa::config::search::core::ProtonConfig::Index &indexCfg,
                                   std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager) const;
 
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
index 1d0bd08ea98..dfe5eaf67a1 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
@@ -13,6 +13,7 @@
 #include <vespa/searchlib/util/filekit.h>
 #include <vespa/vespalib/util/autoclosurecaller.h>
 #include <vespa/vespalib/util/closuretask.h>
+#include <vespa/searchlib/common/lambdatask.h>
 #include <sstream>
 #include <vespa/searchcorespi/flush/closureflushtask.h>
 #include <vespa/vespalib/util/exceptions.h>
@@ -30,6 +31,7 @@ using search::common::FileHeaderContext;
 using search::queryeval::ISourceSelector;
 using search::queryeval::Source;
 using search::SerialNum;
+using search::makeLambdaTask;
 using std::ostringstream;
 using vespalib::makeClosure;
 using vespalib::makeTask;
@@ -869,6 +871,8 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config,
     sourceList->setCurrentIndex(_current_index_id);
     _source_list = std::move(sourceList);
     _fusion_spec = spec;
+    _ctx.getThreadingService().master().execute(makeLambdaTask([this,&config]() {internalWipeHistory(_schema, config.getSerialNum()); }));
+    _ctx.getThreadingService().master().sync();
 }
 
 IndexMaintainer::~IndexMaintainer()
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp
index bb9bed17360..84ce5b77bef 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp
@@ -15,11 +15,13 @@ IndexMaintainerConfig::IndexMaintainerConfig(const vespalib::string &baseDir,
                                              const WarmupConfig & warmup,
                                              size_t maxFlushed,
                                              const Schema &schema,
+                                             const search::SerialNum serialNum,
                                              const TuneFileAttributes &tuneFileAttributes)
     : _baseDir(baseDir),
       _warmup(warmup),
       _maxFlushed(maxFlushed),
       _schema(schema),
+      _serialNum(serialNum),
       _tuneFileAttributes(tuneFileAttributes)
 {
 }
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h
index 9176ae1fd31..ea51399bcd7 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h
@@ -3,6 +3,7 @@
 
 #include <vespa/searchcommon/common/schema.h>
 #include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchlib/common/serialnum.h>
 #include <vespa/vespalib/stllike/string.h>
 #include "warmupconfig.h"
 
@@ -18,6 +19,7 @@ private:
     const WarmupConfig _warmup;
     const size_t _maxFlushed;
     const search::index::Schema _schema;
+    const search::SerialNum _serialNum;
     const search::TuneFileAttributes _tuneFileAttributes;
 
 public:
@@ -25,6 +27,7 @@ public:
                           const WarmupConfig & warmup,
                           size_t maxFlushed,
                           const search::index::Schema &schema,
+                          const search::SerialNum serialNum,
                           const search::TuneFileAttributes &tuneFileAttributes);
 
     /**
@@ -43,6 +46,8 @@ public:
         return _schema;
     }
 
+    search::SerialNum getSerialNum() const { return _serialNum; }
+
     /**
      * Returns the specification on how to read/write attribute vector data files.
      */