diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-06-28 12:58:16 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-06-28 13:07:19 +0000 |
commit | d11299a5a7b96fd1a8e6e52d26cd01ba821cf008 (patch) | |
tree | dba5108ae8270f3b95b73a803840fa6c5faab189 /searchcorespi | |
parent | a71b139ba48f64787d1a37aae636f76b0f0bc550 (diff) |
Ensure that interleaved features can be reconstructed in posting lists by running 2x triggerFlush.
Diffstat (limited to 'searchcorespi')
-rw-r--r-- | searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp | 31 | ||||
-rw-r--r-- | searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h | 1 |
2 files changed, 31 insertions, 1 deletions
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index a174592eb55..083fe70969c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -11,6 +11,7 @@ #include <vespa/fastos/file.h> #include <vespa/searchcorespi/flush/closureflushtask.h> #include <vespa/searchlib/common/serialnumfileheadercontext.h> +#include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/io/fileutil.h> @@ -28,6 +29,7 @@ using document::Document; using search::FixedSourceSelector; using search::TuneFileAttributes; using search::index::Schema; +using search::index::SchemaUtil; using search::common::FileHeaderContext; using search::queryeval::ISourceSelector; using search::queryeval::Source; @@ -466,7 +468,9 @@ IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index) { LockGuard lock(_index_update_lock); if (!_current_index->hasReceivedDocumentInsert() && - _source_selector_changes == 0) { + _source_selector_changes == 0 && + !_flush_empty_current_index) + { args->_skippedEmptyLast = true; // Skip flush of empty memory index } @@ -480,6 +484,7 @@ IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index) _source_selector_changes = 0; } _current_index = *new_index; + _flush_empty_current_index = false; } if (args->_skippedEmptyLast) { replaceSource(_current_index_id, _current_index); @@ -723,6 +728,23 @@ IndexMaintainer::warmupDone(ISearchableIndexCollection::SP current) } } +namespace { + +bool +has_matching_interleaved_features(const Schema& old_schema, const Schema& new_schema) +{ + for (SchemaUtil::IndexIterator itr(new_schema); itr.isValid(); ++itr) { + if (itr.hasMatchingOldFields(old_schema) && + !itr.has_matching_use_interleaved_features(old_schema)) + { + return false; + } + } + return true; +} + +} + void IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex) @@ -758,6 +780,12 @@ IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex) _frozenMemoryIndexes.emplace_back(args._oldIndex, freezeSerialNum, std::move(saveInfo), oldAbsoluteId); } _current_index = newIndex; + // Non-matching interleaved features in schemas means that we need to + // reconstruct or drop interleaved features in posting lists. + // If so, we must flush the new index to disk even if it is empty. + // This ensures that 2x triggerFlush will run fusion + // to reconstruct or drop interleaved features in the posting lists. + _flush_empty_current_index = !has_matching_interleaved_features(args._oldSchema, args._newSchema); } if (dropEmptyLast) { replaceSource(_current_index_id, _current_index); @@ -822,6 +850,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, _next_id(), _current_index_id(), _current_index(), + _flush_empty_current_index(false), _current_serial_num(0), _flush_serial_num(0), _lastFlushTime(), diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index c30825cbb3c..e95613017fa 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -91,6 +91,7 @@ class IndexMaintainer : public IIndexManager, uint32_t _next_id; // Protected by SL + IUL uint32_t _current_index_id; // Protected by SL + IUL IMemoryIndex::SP _current_index; // Protected by SL + IUL + bool _flush_empty_current_index; SerialNum _current_serial_num;// Protected by IUL SerialNum _flush_serial_num; // Protected by SL fastos::TimeStamp _lastFlushTime; // Protected by SL |