From 458ef496df6c381f33f71fe589efb44335fbb9f6 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sat, 4 Aug 2018 15:24:56 +0200 Subject: Sync index dirs before marking index dir valid. Sync config save dir before marking config save valid. --- .../vespa/searchcorespi/index/diskindexcleaner.cpp | 21 ++++++++++++++++----- .../vespa/searchcorespi/index/indexmaintainer.cpp | 2 ++ .../searchcorespi/index/indexwriteutilities.cpp | 8 ++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) (limited to 'searchcorespi') diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp index c363a93815b..e0cb8548f0d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp @@ -3,6 +3,7 @@ #include "diskindexcleaner.h" #include "activediskindexes.h" #include +#include #include #include @@ -34,6 +35,11 @@ bool isValidIndex(const string &index_dir) { return serial_file.OpenReadOnlyExisting(); } +void invalidateIndex(const string &index_dir) { + vespalib::unlink(index_dir + "/serial.dat"); + vespalib::File::sync(index_dir); +} + uint32_t findLastFusionId(const string &base_dir, const vector &indexes) { uint32_t fusion_id = 0; @@ -56,7 +62,8 @@ uint32_t findLastFusionId(const string &base_dir, void removeDir(const string &dir) { LOG(debug, "Removing index dir '%s'", dir.c_str()); - FastOS_FileInterface::EmptyAndRemoveDirectory(dir.c_str()); + invalidateIndex(dir); + vespalib::rmdir(dir, true); } bool isOldIndex(const string &index, uint32_t last_fusion_id) { @@ -73,14 +80,18 @@ bool isOldIndex(const string &index, uint32_t last_fusion_id) { } void removeOld(const string &base_dir, const vector &indexes, - const ActiveDiskIndexes &active_indexes) { + const ActiveDiskIndexes &active_indexes, bool remove) { uint32_t last_fusion_id = findLastFusionId(base_dir, indexes); for (size_t i = 0; i < indexes.size(); ++i) { const string index_dir = base_dir + "/" + indexes[i]; if (isOldIndex(indexes[i], last_fusion_id) && !active_indexes.isActive(index_dir)) { - removeDir(index_dir); + if (remove) { + removeDir(index_dir); + } else { + invalidateIndex(index_dir); + } } } } @@ -99,14 +110,14 @@ void removeInvalid(const string &base_dir, const vector &indexes) { void DiskIndexCleaner::clean(const string &base_dir, const ActiveDiskIndexes &active_indexes) { vector indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes); + removeOld(base_dir, indexes, active_indexes, false); removeInvalid(base_dir, indexes); } void DiskIndexCleaner::removeOldIndexes( const string &base_dir, const ActiveDiskIndexes &active_indexes) { vector indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes); + removeOld(base_dir, indexes, active_indexes, true); } } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index a18d24931cb..1e63741084d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -999,6 +1000,7 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec) _activeFusionSchema.reset(); _activeFusionPrunedSchema.reset(); } + vespalib::File::sync(vespalib::dirname(fail_dir)); return fusion_spec.last_fusion_id; } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp index c8b2e81a9c0..ebb316610e1 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum, tmpFileName.c_str()); } file.Close(); + vespalib::File::sync(dir); if (ok) { FastOS_File renameFile(tmpFileName.c_str()); @@ -67,6 +69,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum, msg << "Unable to write serial number to '" << dir << "'."; throw IllegalStateException(msg.str()); } + vespalib::File::sync(dir); } bool @@ -94,12 +97,14 @@ IndexWriteUtilities::copySerialNumFile(const vespalib::string &sourceDir, return false; } file.Close(); + vespalib::File::sync(destDir); if (!file.Rename(dest.c_str())) { LOG(error, "Unable to rename file '%s' to '%s'", tmpDest.c_str(), dest.c_str()); return false; } + vespalib::File::sync(destDir); return true; } @@ -151,6 +156,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, } vespalib::string schemaTmpName = schemaName + ".tmp"; vespalib::string schemaOrigName = schemaName + ".orig"; + vespalib::unlink(schemaTmpName); if (!newSchema->saveToFile(schemaTmpName)) { LOG(error, "Could not save schema to '%s'", schemaTmpName.c_str()); @@ -172,6 +178,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, schemaName.c_str(), FastOS_File::getLastErrorString().c_str()); } + vespalib::File::sync(indexDir); } // XXX: FastOS layer violation int renameres = ::rename(schemaTmpName.c_str(), schemaName.c_str()); @@ -183,6 +190,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, schemaName.c_str(), errString.c_str()); } + vespalib::File::sync(indexDir); } } // namespace index -- cgit v1.2.3