diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2019-06-28 11:32:57 -0500 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2019-06-28 11:32:57 -0500 |
commit | 609ab45a9eded61cef79eeb1941b6f06cb713cb6 (patch) | |
tree | fbc07799e06f982d53167ff2021e5a9329078a9c | |
parent | 781d6dfd8afdcb1da0ce29ef37ccbc0c45dc155d (diff) | |
parent | e5d5699d6b0964cc97aacf77cccb0159d6299284 (diff) |
Merge with master
5 files changed, 53 insertions, 8 deletions
diff --git a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java index f23e1f33d8f..3e720f9e0aa 100644 --- a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java @@ -10,6 +10,7 @@ import com.yahoo.document.Field; import com.yahoo.document.annotation.Annotation; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.MapFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.StructuredFieldValue; @@ -55,11 +56,16 @@ public abstract class AbstractConcreteDocumentFactory extends com.yahoo.componen } else if (fv instanceof Array) { Array<FieldValue> array = (Array<FieldValue>) fv; DataType nestedType = array.getDataType().getNestedType(); - for (int i=0; i < array.size(); i++) { - array.set(i, optionallyUpgrade(nestedType, array.get(i))); + if (nestedType.getPrimitiveType() == null) { + array.replaceAll((item) -> optionallyUpgrade(nestedType, item)); + } + } else if (fv instanceof MapFieldValue) { + MapFieldValue<FieldValue, FieldValue> map = (MapFieldValue<FieldValue, FieldValue>) fv; + DataType valueTypeType = map.getDataType().getValueType(); + if (valueTypeType.getPrimitiveType() == null) { + map.replaceAll((key, value) -> optionallyUpgrade(valueTypeType, value)); } } - // TODO We also need special handling for weighted set/map. Limiting to array until verified. return fv; } diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 25277e9cdfe..1e8c585b43e 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -321,9 +321,7 @@ public class DocumentGenPluginTest { verifyArrayOfStruct(toBook(copyBySerialization(book))); } - @Test - public void testMaps() { - Book book = getBook(); + private void verifyMaps(Book book) { assertTrue(book.getField("stringmap").getDataType() instanceof MapDataType); MapFieldValue mfv = (MapFieldValue) book.getFieldValue("stringmap"); assertEquals(mfv.get(new StringFieldValue("Melville")), new StringFieldValue("Moby Dick")); @@ -333,6 +331,8 @@ public class DocumentGenPluginTest { assertEquals(mfv.keySet().size(), 2); book.getStringmap().put("Melville", "MD"); assertEquals(mfv.keySet().size(), 3); + book.getStringmap().put("Melville", "Moby Dick"); + assertEquals(mfv.keySet().size(), 3); assertEquals(book.getStructmap().get(50).getS1(), "test s1"); MapFieldValue mfv2 = (MapFieldValue) book.getFieldValue("structmap"); @@ -342,6 +342,13 @@ public class DocumentGenPluginTest { } @Test + public void testMaps() { + Book book = getBook(); + verifyMaps(book); + verifyMaps(toBook(copyBySerialization(book))); + } + + @Test public void testWeightedSets() { Book book = getBook(); assertTrue(book.getField("mywsfloat").getDataType() instanceof WeightedSetDataType); diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index a174592eb55..083fe70969c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -11,6 +11,7 @@ #include <vespa/fastos/file.h> #include <vespa/searchcorespi/flush/closureflushtask.h> #include <vespa/searchlib/common/serialnumfileheadercontext.h> +#include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/io/fileutil.h> @@ -28,6 +29,7 @@ using document::Document; using search::FixedSourceSelector; using search::TuneFileAttributes; using search::index::Schema; +using search::index::SchemaUtil; using search::common::FileHeaderContext; using search::queryeval::ISourceSelector; using search::queryeval::Source; @@ -466,7 +468,9 @@ IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index) { LockGuard lock(_index_update_lock); if (!_current_index->hasReceivedDocumentInsert() && - _source_selector_changes == 0) { + _source_selector_changes == 0 && + !_flush_empty_current_index) + { args->_skippedEmptyLast = true; // Skip flush of empty memory index } @@ -480,6 +484,7 @@ IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index) _source_selector_changes = 0; } _current_index = *new_index; + _flush_empty_current_index = false; } if (args->_skippedEmptyLast) { replaceSource(_current_index_id, _current_index); @@ -723,6 +728,23 @@ IndexMaintainer::warmupDone(ISearchableIndexCollection::SP current) } } +namespace { + +bool +has_matching_interleaved_features(const Schema& old_schema, const Schema& new_schema) +{ + for (SchemaUtil::IndexIterator itr(new_schema); itr.isValid(); ++itr) { + if (itr.hasMatchingOldFields(old_schema) && + !itr.has_matching_use_interleaved_features(old_schema)) + { + return false; + } + } + return true; +} + +} + void IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex) @@ -758,6 +780,12 @@ IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex) _frozenMemoryIndexes.emplace_back(args._oldIndex, freezeSerialNum, std::move(saveInfo), oldAbsoluteId); } _current_index = newIndex; + // Non-matching interleaved features in schemas means that we need to + // reconstruct or drop interleaved features in posting lists. + // If so, we must flush the new index to disk even if it is empty. + // This ensures that 2x triggerFlush will run fusion + // to reconstruct or drop interleaved features in the posting lists. + _flush_empty_current_index = !has_matching_interleaved_features(args._oldSchema, args._newSchema); } if (dropEmptyLast) { replaceSource(_current_index_id, _current_index); @@ -822,6 +850,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, _next_id(), _current_index_id(), _current_index(), + _flush_empty_current_index(false), _current_serial_num(0), _flush_serial_num(0), _lastFlushTime(), diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index c30825cbb3c..e95613017fa 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -91,6 +91,7 @@ class IndexMaintainer : public IIndexManager, uint32_t _next_id; // Protected by SL + IUL uint32_t _current_index_id; // Protected by SL + IUL IMemoryIndex::SP _current_index; // Protected by SL + IUL + bool _flush_empty_current_index; SerialNum _current_serial_num;// Protected by IUL SerialNum _flush_serial_num; // Protected by SL fastos::TimeStamp _lastFlushTime; // Protected by SL diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index d349164e8ed..124e8c607ee 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -6,6 +6,7 @@ #include "distributor_bucket_space.h" #include "distributormetricsset.h" #include "simpleclusterinformation.h" +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> @@ -142,7 +143,8 @@ BucketDBUpdater::removeSuperfluousBuckets( if (!is_distribution_config_change && db_pruning_may_be_elided(oldClusterState, *new_cluster_state, up_states)) { - LOG(debug, "Eliding DB pruning for state transition '%s' -> '%s'", + LOG(info, "[bucket space '%s']: eliding DB pruning for state transition '%s' -> '%s'", + document::FixedBucketSpaces::to_string(elem.first).data(), oldClusterState.toString().c_str(), new_cluster_state->toString().c_str()); continue; } |