Sync index dirs before marking index dir valid.

Sync config save dir before marking config save valid.

9 files changed, 35 insertions, 5 deletions

diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
index bb2f99d077b..a675927b85f 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
@@ -14,6 +14,7 @@ AttributeDiskLayout::AttributeDiskLayout(const vespalib::string &baseDir, Privat
       _dirs()
 {
     vespalib::mkdir(_baseDir, false);
+    vespalib::File::sync(vespalib::dirname(_baseDir));
 }
 
 AttributeDiskLayout::~AttributeDiskLayout()
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
index b8f2947c9b4..ad233a66d1f 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
@@ -45,6 +45,7 @@ IndexManagerInitializer::run()
     LOG(debug, "About to create proton::IndexManager with %u index field(s)",
         _schema.getNumIndexFields());
     vespalib::mkdir(_baseDir, false);
+    vespalib::File::sync(vespalib::dirname(_baseDir));
     *_indexManager = std::make_shared<proton::IndexManager>
                     (_baseDir,
                      _warmupCfg,
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
index 76f2ffce93a..72f48df3295 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
@@ -229,6 +229,7 @@ FileConfigManager::FileConfigManager(const vespalib::string &baseDir,
       _protonConfig()
 {
     vespalib::mkdir(baseDir, false);
+    vespalib::File::sync(vespalib::dirname(baseDir));
     if (!_info.load())
         _info.save();
     removeInvalid();
@@ -297,6 +298,8 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot,
     bool saveHistorySchemaRes = historySchema.saveToFile(snapDir + "/historyschema.txt");
     assert(saveHistorySchemaRes);
     (void) saveHistorySchemaRes;
+    vespalib::File::sync(snapDir);
+    vespalib::File::sync(_baseDir);
 
     _info.validateSnapshot(serialNum);
 
@@ -402,6 +405,7 @@ FileConfigManager::removeInvalid()
             LOG(warning, "Removing obsolete config directory '%s' failed due to %s", snapDir.c_str(), e.what());
         }
     }
+    vespalib::File::sync(_baseDir);
     for (const auto &serial : toRem) {
         _info.removeSnapshot(serial);
     }
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index ad06525a5a3..2d3c204d259 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -139,6 +139,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx)
       _gidToLidChangeHandler(std::make_shared<DummyGidToLidChangeHandler>())
 {
     vespalib::mkdir(_baseDir, false); // Assume parent is created.
+    vespalib::File::sync(vespalib::dirname(_baseDir));
 }
 
 StoreOnlyDocSubDB::~StoreOnlyDocSubDB()
diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp
index c363a93815b..e0cb8548f0d 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp
@@ -3,6 +3,7 @@
 #include "diskindexcleaner.h"
 #include "activediskindexes.h"
 #include <vespa/fastos/file.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <sstream>
 #include <vector>
 
@@ -34,6 +35,11 @@ bool isValidIndex(const string &index_dir) {
     return serial_file.OpenReadOnlyExisting();
 }
 
+void invalidateIndex(const string &index_dir) {
+    vespalib::unlink(index_dir + "/serial.dat");
+    vespalib::File::sync(index_dir);
+}
+
 uint32_t findLastFusionId(const string &base_dir,
                           const vector<string> &indexes) {
     uint32_t fusion_id = 0;
@@ -56,7 +62,8 @@ uint32_t findLastFusionId(const string &base_dir,
 
 void removeDir(const string &dir) {
     LOG(debug, "Removing index dir '%s'", dir.c_str());
-    FastOS_FileInterface::EmptyAndRemoveDirectory(dir.c_str());
+    invalidateIndex(dir);
+    vespalib::rmdir(dir, true);
 }
 
 bool isOldIndex(const string &index, uint32_t last_fusion_id) {
@@ -73,14 +80,18 @@ bool isOldIndex(const string &index, uint32_t last_fusion_id) {
 }
 
 void removeOld(const string &base_dir, const vector<string> &indexes,
-               const ActiveDiskIndexes &active_indexes) {
+               const ActiveDiskIndexes &active_indexes, bool remove) {
     uint32_t last_fusion_id = findLastFusionId(base_dir, indexes);
     for (size_t i = 0; i < indexes.size(); ++i) {
         const string index_dir = base_dir + "/" + indexes[i];
         if (isOldIndex(indexes[i], last_fusion_id) &&
             !active_indexes.isActive(index_dir))
         {
-            removeDir(index_dir);
+            if (remove) {
+                removeDir(index_dir);
+            } else {
+                invalidateIndex(index_dir);
+            }
         }
     }
 }
@@ -99,14 +110,14 @@ void removeInvalid(const string &base_dir, const vector<string> &indexes) {
 void DiskIndexCleaner::clean(const string &base_dir,
                              const ActiveDiskIndexes &active_indexes) {
     vector<string> indexes = readIndexes(base_dir);
-    removeOld(base_dir, indexes, active_indexes);
+    removeOld(base_dir, indexes, active_indexes, false);
     removeInvalid(base_dir, indexes);
 }
 
 void DiskIndexCleaner::removeOldIndexes(
         const string &base_dir, const ActiveDiskIndexes &active_indexes) {
     vector<string> indexes = readIndexes(base_dir);
-    removeOld(base_dir, indexes, active_indexes);
+    removeOld(base_dir, indexes, active_indexes, true);
 }
 
 }
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
index a18d24931cb..1e63741084d 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
@@ -16,6 +16,7 @@
 #include <vespa/vespalib/util/lambdatask.h>
 #include <sstream>
 #include <vespa/searchcorespi/flush/closureflushtask.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/array.hpp>
 #include <vespa/fastos/file.h>
@@ -999,6 +1000,7 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec)
             _activeFusionSchema.reset();
             _activeFusionPrunedSchema.reset();
         }
+        vespalib::File::sync(vespalib::dirname(fail_dir));
         return fusion_spec.last_fusion_id;
     }
 
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp
index c8b2e81a9c0..ebb316610e1 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchlib/common/serialnumfileheadercontext.h>
 #include <vespa/searchlib/index/schemautil.h>
 #include <vespa/fastlib/io/bufferedfile.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <sstream>
 #include <unistd.h>
@@ -57,6 +58,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum,
             tmpFileName.c_str());
     }
     file.Close();
+    vespalib::File::sync(dir);
 
     if (ok) {
         FastOS_File renameFile(tmpFileName.c_str());
@@ -67,6 +69,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum,
         msg << "Unable to write serial number to '" << dir << "'.";
         throw IllegalStateException(msg.str());
     }
+    vespalib::File::sync(dir);
 }
 
 bool
@@ -94,12 +97,14 @@ IndexWriteUtilities::copySerialNumFile(const vespalib::string &sourceDir,
         return false;
     }
     file.Close();
+    vespalib::File::sync(destDir);
     if (!file.Rename(dest.c_str())) {
         LOG(error,
             "Unable to rename file '%s' to '%s'",
             tmpDest.c_str(), dest.c_str());
         return false;
     }
+    vespalib::File::sync(destDir);
     return true;
 }
 
@@ -151,6 +156,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir,
     }
     vespalib::string schemaTmpName = schemaName + ".tmp";
     vespalib::string schemaOrigName = schemaName + ".orig";
+    vespalib::unlink(schemaTmpName);
     if (!newSchema->saveToFile(schemaTmpName)) {
         LOG(error, "Could not save schema to '%s'",
             schemaTmpName.c_str());
@@ -172,6 +178,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir,
                 schemaName.c_str(),
                 FastOS_File::getLastErrorString().c_str());
         }
+        vespalib::File::sync(indexDir);
     }
     // XXX: FastOS layer violation
     int renameres = ::rename(schemaTmpName.c_str(), schemaName.c_str());
@@ -183,6 +190,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir,
             schemaName.c_str(),
             errString.c_str());
     }
+    vespalib::File::sync(indexDir);
 }
 
 } // namespace index
diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
index e61fe7bab17..2db6025d257 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
@@ -218,6 +218,7 @@ Fusion::mergeField(uint32_t id)
     }
     if (!FileKit::createStamp(indexDir +  "/.mergeocc_done"))
         return false;
+    vespalib::File::sync(indexDir);
 
     if (!CleanTmpDirs())
         return false;
diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
index b879e1f65a6..b744a68932d 100644
--- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
@@ -700,6 +700,7 @@ IndexBuilder::close()
     for (FieldHandle & fh : _fields) {
         if (fh.getValid()) {
             fh.close();
+            vespalib::File::sync(fh.getDir());
         }
     }
     if (!docsummary::DocumentSummary::writeDocIdLimit(_prefix, _docIdLimit)) {