Simplify wipe history arguments to attribute manager, supply serial number

for history wipe instead of history schema.

Don't wipe attribute that will be resurrected later on during replay of
transaction log.

19 files changed, 60 insertions, 37 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index afd2150a1bd..ad694348029 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -523,10 +523,12 @@ TEST_F("require that history can be wiped", Fixture)
     f.addAttribute("a2");
     f.addAttribute("a3");
     f._m.flushAll(10);
-    Schema hs;
-    hs.addAttributeField(Schema::AttributeField("a1", schema::INT32));
-    hs.addAttributeField(Schema::AttributeField("a3", schema::INT32));
-    f._m.wipeHistory(hs);
+
+    AttrSpecList newSpec;
+    newSpec.push_back(AttrSpec("a2", INT32_SINGLE));
+    SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 1, 11));
+    sam.mgr.wipeHistory(11);
+
     FastOS_StatInfo si;
     EXPECT_TRUE(!FastOS_File::Stat(vespalib::string(test_dir + "/a1").c_str(), &si));
     EXPECT_TRUE(FastOS_File::Stat(vespalib::string(test_dir + "/a2").c_str(), &si));
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index 910ca286b79..ffd698691b4 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -375,7 +375,7 @@ struct MyAttributeManager : public proton::IAttributeManager
     virtual void getAttributeListAll(std::vector<search::AttributeGuard> &) const override {
         abort();
     }
-    virtual void wipeHistory(const search::index::Schema &) override {
+    virtual void wipeHistory(search::SerialNum) override {
         abort();
     }
     virtual const IAttributeFactory::SP &getFactory() const override {
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
index a4fb2e03c44..c9e1f7704b0 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
@@ -123,9 +123,15 @@ AttributeDiskLayout::removeOldSnapshots(IndexMetaInfo &snapInfo,
 
 bool
 AttributeDiskLayout::removeAttribute(const vespalib::string &baseDir,
-                                     const vespalib::string &attrName)
+                                     const vespalib::string &attrName,
+                                     uint64_t wipeSerial)
 {
     const vespalib::string currDir = getAttributeBaseDir(baseDir, attrName);
+    search::IndexMetaInfo snapInfo(currDir);
+    IndexMetaInfo::Snapshot best = snapInfo.getBestSnapshot();
+    if (best.valid && best.syncToken >= wipeSerial) {
+        return true; // Attribute has been resurrected and flushed later on
+    }
     const vespalib::string rmDir =
         getAttributeBaseDir(baseDir,
                             vespalib::make_string("remove.%s",
@@ -168,6 +174,22 @@ AttributeDiskLayout::removeAttribute(const vespalib::string &baseDir,
     return true;
 }
 
+std::vector<vespalib::string>
+AttributeDiskLayout::listAttributes(const vespalib::string &baseDir)
+{
+    std::vector<vespalib::string> attributes;
+    FastOS_DirectoryScan dir(baseDir.c_str());
+    while (dir.ReadNext()) {
+        if (strcmp(dir.GetName(), "..") != 0 &&
+            strcmp(dir.GetName(), ".") != 0)
+        {
+            if (dir.IsDirectory()) {
+                attributes.emplace_back(dir.GetName());
+            }
+        }
+    }
+    return attributes;
+}
 
 } // namespace proton
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h
index f085d9c6f61..7db932fcf69 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h
@@ -22,7 +22,8 @@ public:
     static vespalib::string  getAttributeBaseDir(const vespalib::string &baseDir, const vespalib::string &attrName);
     static search::AttributeVector::BaseName getAttributeFileName(const vespalib::string &baseDir, const vespalib::string &attrName, uint64_t syncToken);
     static bool removeOldSnapshots(search::IndexMetaInfo &snapInfo, vespalib::Lock &snapInfoLock);
-    static bool removeAttribute(const vespalib::string &baseDir, const vespalib::string &attrName);
+    static bool removeAttribute(const vespalib::string &baseDir, const vespalib::string &attrName, uint64_t wipeSerial);
+    static std::vector<vespalib::string> listAttributes(const vespalib::string &baseDir);
 
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
index 19a2d02834d..f52153f6135 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -445,12 +445,14 @@ AttributeManager::getAttributeListAll(std::vector<AttributeGuard> &list) const
 }
 
 void
-AttributeManager::wipeHistory(const Schema &historySchema)
+AttributeManager::wipeHistory(search::SerialNum wipeSerial)
 {
-    for (uint32_t i = 0; i < historySchema.getNumAttributeFields(); ++i) {
-        const Schema::AttributeField & field =
-            historySchema.getAttributeField(i);
-        AttributeDiskLayout::removeAttribute(_baseDir, field.getName());
+    std::vector<vespalib::string> attributes = AttributeDiskLayout::listAttributes(_baseDir);
+    for (const auto &attribute : attributes) {
+        auto itr = _attributes.find(attribute);
+        if (itr == _attributes.end()) {
+            AttributeDiskLayout::removeAttribute(_baseDir, attribute, wipeSerial);
+        }
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
index 0462edcfc1b..3a9405f5773 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
@@ -165,7 +165,7 @@ public:
 
     virtual void getAttributeListAll(std::vector<search::AttributeGuard> &list) const override;
 
-    virtual void wipeHistory(const search::index::Schema &historySchema) override;
+    virtual void wipeHistory(search::SerialNum wipeSerial) override;
 
     virtual const IAttributeFactory::SP &getFactory() const override { return _factory; }
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
index c0fca16af52..d6540331345 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
@@ -60,7 +60,7 @@ FilterAttributeManager::getAttributeListAll(std::vector<search::AttributeGuard>
     throw vespalib::IllegalArgumentException("Not implemented");
 }
 void
-FilterAttributeManager::wipeHistory(const search::index::Schema &) {
+FilterAttributeManager::wipeHistory(search::SerialNum) {
     throw vespalib::IllegalArgumentException("Not implemented");
 }
 const IAttributeFactory::SP &
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
index 4c374abba2d..05b842ee30a 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
@@ -44,7 +44,7 @@ public:
     virtual search::SerialNum getOldestFlushedSerialNumber() const;
     virtual search::SerialNum getNewestFlushedSerialNumber() const;
     virtual void getAttributeListAll(std::vector<search::AttributeGuard> &) const;
-    virtual void wipeHistory(const search::index::Schema &);
+    virtual void wipeHistory(search::SerialNum wipeSerial) override;
     virtual const IAttributeFactory::SP &getFactory() const;
     virtual search::ISequencedTaskExecutor & getAttributeFieldWriter() const override;
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
index f185e82846a..1935e85dcc4 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
@@ -67,9 +67,9 @@ struct IAttributeManager : public search::IAttributeManager
     virtual void getAttributeListAll(std::vector<search::AttributeGuard> &list) const = 0;
 
     /**
-     * Wipe history using the given schema.
+     * Wipe unknown attributes from file system.
      */
-    virtual void wipeHistory(const search::index::Schema &historySchema) = 0;
+    virtual void wipeHistory(search::SerialNum wipeSerial) = 0;
 
     /**
      * Returns the attribute factory used by this manager.
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 61799861171..98ca479e8c8 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -993,7 +993,7 @@ DocumentDB::performWipeHistory()
         Schema::UP newHistory(new Schema);
         writeWipeHistoryTransactionLogEntry(wipeSerial, 0,
                                             *configSnapshot, *newHistory);
-        internalWipeHistory(wipeSerial, std::move(newHistory), *_historySchema);
+        internalWipeHistory(wipeSerial, std::move(newHistory));
     }
 }
 
@@ -1013,11 +1013,10 @@ void DocumentDB::writeWipeHistoryTransactionLogEntry(
 
 void
 DocumentDB::internalWipeHistory(SerialNum wipeSerial,
-                                Schema::UP newHistorySchema,
-                                const Schema &wipeSchema)
+                                Schema::UP newHistorySchema)
 {
     // Called by executor thread
-    _subDBs.wipeHistory(wipeSerial, wipeSchema);
+    _subDBs.wipeHistory(wipeSerial);
     _historySchema.reset(newHistorySchema.release());
 }
 
@@ -1074,7 +1073,7 @@ DocumentDB::replayWipeHistory(search::SerialNum serialNum,
     }
     LOG(info, "DocumentDB(%s): Replayed history wipe with serialNum=%" PRIu64,
         _docTypeName.toString().c_str(), serialNum);
-    internalWipeHistory(serialNum, std::move(newHistory), *wipeSchema);
+    internalWipeHistory(serialNum, std::move(newHistory));
 }
 
 
@@ -1287,7 +1286,7 @@ DocumentDB::wipeOldRemovedFields(fastos::TimeStamp wipeTimeLimit)
     SerialNum wipeSerial = _feedHandler.incSerialNum();
     writeWipeHistoryTransactionLogEntry(wipeSerial, wipeTimeLimit,
                                         *configSnapshot, *newHistorySchema);
-    internalWipeHistory(wipeSerial, std::move(newHistorySchema), *wipeSchema);
+    internalWipeHistory(wipeSerial, std::move(newHistorySchema));
 
     LOG(debug, "DocumentDB(%s): Done wipeOldRemovedFields: wipe(%s), history(%s) timeLimit(%" PRIu64 ")",
         _docTypeName.toString().c_str(),
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index ec1c7963dd8..3b6c335f1e6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -227,7 +227,7 @@ private:
             const DocumentDBConfig &configSnapshot,
             const Schema &newHistorySchema);
 
-    void internalWipeHistory(SerialNum wipeSerial, Schema::UP newHistorySchema, const Schema &wipeSchema);
+    void internalWipeHistory(SerialNum wipeSerial, Schema::UP newHistorySchema);
 
     void startTransactionLogReplay();
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 5582571ba74..5dd4520e2b0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -246,11 +246,10 @@ DocumentSubDBCollection::getNewestFlushedSerial()
 
 
 void
-DocumentSubDBCollection::wipeHistory(SerialNum wipeSerial,
-                                     const Schema &wipeSchema)
+DocumentSubDBCollection::wipeHistory(SerialNum wipeSerial)
 {
     for (auto subDb : _subDBs) {
-        subDb->wipeHistory(wipeSerial, wipeSchema);
+        subDb->wipeHistory(wipeSerial);
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index cc856fbc4aa..5114c656231 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -110,8 +110,7 @@ public:
     SerialNum getNewestFlushedSerial(void);
 
     void
-    wipeHistory(SerialNum wipeSerial,
-                const search::index::Schema &wipeSchema);
+    wipeHistory(SerialNum wipeSerial);
 
     void
     applyConfig(const DocumentDBConfig &newConfigSnapshot,
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
index b0e73bc5c72..72fad6f621c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
@@ -122,7 +122,7 @@ public:
      * last part of transaction log.
      */
     virtual SerialNum getNewestFlushedSerial()  = 0;
-    virtual void wipeHistory(SerialNum wipeSerial, const Schema &wipeSchema) = 0;
+    virtual void wipeHistory(SerialNum wipeSerial) = 0;
     virtual void setIndexSchema(const SchemaSP &schema, SerialNum serialNum) = 0;
     virtual search::SearchableStats getSearchableStats() const = 0;
     virtual std::unique_ptr<IDocumentRetriever> getDocumentRetriever() = 0;
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index 2713b61880b..cec96386d15 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -297,15 +297,14 @@ SearchableDocSubDB::getFlushTargetsInternal()
 }
 
 void
-SearchableDocSubDB::wipeHistory(SerialNum wipeSerial,
-                                const Schema &wipeSchema)
+SearchableDocSubDB::wipeHistory(SerialNum wipeSerial)
 {
     assert(_writeService.master().isCurrentThread());
     SearchView::SP oldSearchView = _rSearchView.get();
     IFeedView::SP oldFeedView = _iFeedView.get();
     _indexMgr->wipeHistory(wipeSerial);
     reconfigureIndexSearchable();
-    getAttributeManager()->wipeHistory(wipeSchema);
+    getAttributeManager()->wipeHistory(wipeSerial);
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index 9d2b45fa3ac..3beb7441000 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -154,7 +154,7 @@ public:
 
     SerialNum getOldestFlushedSerial() override;
     SerialNum getNewestFlushedSerial() override;
-    void wipeHistory(SerialNum wipeSerial, const Schema &wipeSchema) override;
+    void wipeHistory(SerialNum wipeSerial) override;
     void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override;
     size_t getNumActiveDocs() const override;
     search::SearchableStats getSearchableStats() const override ;
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index 06bcd509522..d6f0cd179dc 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -458,7 +458,7 @@ StoreOnlyDocSubDB::getIndexWriter() const
 }
 
 void
-StoreOnlyDocSubDB::wipeHistory(SerialNum, const Schema &)
+StoreOnlyDocSubDB::wipeHistory(SerialNum)
 {
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index 11b0db61576..44ad409cfdb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -249,7 +249,7 @@ public:
     SerialNum getOldestFlushedSerial() override;
     SerialNum getNewestFlushedSerial() override;
 
-    void wipeHistory(SerialNum wipeSerial, const Schema &wipeSchema) override;
+    void wipeHistory(SerialNum wipeSerial) override;
     void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override;
     search::SearchableStats getSearchableStats() const override;
     IDocumentRetriever::UP getDocumentRetriever() override;
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
index e2638b617b3..42f4a4dc0cf 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
@@ -81,7 +81,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB
     void onReprocessDone(SerialNum) override { }
     SerialNum getOldestFlushedSerial() override { return 0; }
     SerialNum getNewestFlushedSerial() override { return 0; }
-    void wipeHistory(SerialNum, const Schema &) override { }
+    void wipeHistory(SerialNum) override { }
     void setIndexSchema(const Schema::SP &, SerialNum) override { }
     search::SearchableStats getSearchableStats() const override {
         return search::SearchableStats();