Merge with master

5 files changed, 53 insertions, 8 deletions

diff --git a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java
index f23e1f33d8f..3e720f9e0aa 100644
--- a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java
+++ b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java
@@ -10,6 +10,7 @@ import com.yahoo.document.Field;
 import com.yahoo.document.annotation.Annotation;
 import com.yahoo.document.datatypes.Array;
 import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
 import com.yahoo.document.datatypes.Struct;
 import com.yahoo.document.datatypes.StructuredFieldValue;
 
@@ -55,11 +56,16 @@ public abstract class AbstractConcreteDocumentFactory extends com.yahoo.componen
         } else if (fv instanceof Array) {
             Array<FieldValue> array = (Array<FieldValue>) fv;
             DataType nestedType = array.getDataType().getNestedType();
-            for (int i=0; i < array.size(); i++) {
-                array.set(i, optionallyUpgrade(nestedType, array.get(i)));
+            if (nestedType.getPrimitiveType() == null) {
+                array.replaceAll((item) -> optionallyUpgrade(nestedType, item));
+            }
+        } else if (fv instanceof MapFieldValue) {
+            MapFieldValue<FieldValue, FieldValue> map = (MapFieldValue<FieldValue, FieldValue>) fv;
+            DataType valueTypeType = map.getDataType().getValueType();
+            if (valueTypeType.getPrimitiveType() == null) {
+                map.replaceAll((key, value) -> optionallyUpgrade(valueTypeType, value));
             }
         }
-        // TODO We also need special handling for weighted set/map. Limiting to array until verified.
         return fv;
     }
 
diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
index 25277e9cdfe..1e8c585b43e 100644
--- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
+++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
@@ -321,9 +321,7 @@ public class DocumentGenPluginTest {
         verifyArrayOfStruct(toBook(copyBySerialization(book)));
     }
 
-    @Test
-    public void testMaps() {
-        Book book = getBook();
+    private void verifyMaps(Book book) {
         assertTrue(book.getField("stringmap").getDataType() instanceof MapDataType);
         MapFieldValue mfv = (MapFieldValue) book.getFieldValue("stringmap");
         assertEquals(mfv.get(new StringFieldValue("Melville")), new StringFieldValue("Moby Dick"));
@@ -333,6 +331,8 @@ public class DocumentGenPluginTest {
         assertEquals(mfv.keySet().size(), 2);
         book.getStringmap().put("Melville", "MD");
         assertEquals(mfv.keySet().size(), 3);
+        book.getStringmap().put("Melville", "Moby Dick");
+        assertEquals(mfv.keySet().size(), 3);
 
         assertEquals(book.getStructmap().get(50).getS1(), "test s1");
         MapFieldValue mfv2 = (MapFieldValue) book.getFieldValue("structmap");
@@ -342,6 +342,13 @@ public class DocumentGenPluginTest {
     }
 
     @Test
+    public void testMaps() {
+        Book book = getBook();
+        verifyMaps(book);
+        verifyMaps(toBook(copyBySerialization(book)));
+    }
+
+    @Test
     public void testWeightedSets() {
         Book book = getBook();
         assertTrue(book.getField("mywsfloat").getDataType() instanceof WeightedSetDataType);
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
index a174592eb55..083fe70969c 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
@@ -11,6 +11,7 @@
 #include <vespa/fastos/file.h>
 #include <vespa/searchcorespi/flush/closureflushtask.h>
 #include <vespa/searchlib/common/serialnumfileheadercontext.h>
+#include <vespa/searchlib/index/schemautil.h>
 #include <vespa/searchlib/util/dirtraverse.h>
 #include <vespa/searchlib/util/filekit.h>
 #include <vespa/vespalib/io/fileutil.h>
@@ -28,6 +29,7 @@ using document::Document;
 using search::FixedSourceSelector;
 using search::TuneFileAttributes;
 using search::index::Schema;
+using search::index::SchemaUtil;
 using search::common::FileHeaderContext;
 using search::queryeval::ISourceSelector;
 using search::queryeval::Source;
@@ -466,7 +468,9 @@ IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index)
     {
         LockGuard lock(_index_update_lock);
         if (!_current_index->hasReceivedDocumentInsert() &&
-            _source_selector_changes == 0) {
+            _source_selector_changes == 0 &&
+            !_flush_empty_current_index)
+        {
             args->_skippedEmptyLast = true; // Skip flush of empty memory index
         }
 
@@ -480,6 +484,7 @@ IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index)
             _source_selector_changes = 0;
         }
         _current_index = *new_index;
+        _flush_empty_current_index = false;
     }
     if (args->_skippedEmptyLast) {
         replaceSource(_current_index_id, _current_index);
@@ -723,6 +728,23 @@ IndexMaintainer::warmupDone(ISearchableIndexCollection::SP current)
     }
 }
 
+namespace {
+
+bool
+has_matching_interleaved_features(const Schema& old_schema, const Schema& new_schema)
+{
+    for (SchemaUtil::IndexIterator itr(new_schema); itr.isValid(); ++itr) {
+        if (itr.hasMatchingOldFields(old_schema) &&
+                !itr.has_matching_use_interleaved_features(old_schema))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+}
+
 
 void
 IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex)
@@ -758,6 +780,12 @@ IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex)
             _frozenMemoryIndexes.emplace_back(args._oldIndex, freezeSerialNum, std::move(saveInfo), oldAbsoluteId);
         }
         _current_index = newIndex;
+        // Non-matching interleaved features in schemas means that we need to
+        // reconstruct or drop interleaved features in posting lists.
+        // If so, we must flush the new index to disk even if it is empty.
+        // This ensures that 2x triggerFlush will run fusion
+        // to reconstruct or drop interleaved features in the posting lists.
+        _flush_empty_current_index = !has_matching_interleaved_features(args._oldSchema, args._newSchema);
     }
     if (dropEmptyLast) {
         replaceSource(_current_index_id, _current_index);
@@ -822,6 +850,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config,
       _next_id(),
       _current_index_id(),
       _current_index(),
+      _flush_empty_current_index(false),
       _current_serial_num(0),
       _flush_serial_num(0),
       _lastFlushTime(),
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h
index c30825cbb3c..e95613017fa 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h
@@ -91,6 +91,7 @@ class IndexMaintainer : public IIndexManager,
     uint32_t          _next_id;          // Protected by SL + IUL
     uint32_t          _current_index_id; // Protected by SL + IUL
     IMemoryIndex::SP  _current_index;    // Protected by SL + IUL
+    bool              _flush_empty_current_index;
     SerialNum         _current_serial_num;// Protected by IUL
     SerialNum         _flush_serial_num;  // Protected by SL
     fastos::TimeStamp _lastFlushTime; // Protected by SL
diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
index d349164e8ed..124e8c607ee 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
@@ -6,6 +6,7 @@
 #include "distributor_bucket_space.h"
 #include "distributormetricsset.h"
 #include "simpleclusterinformation.h"
+#include <vespa/document/bucket/fixed_bucket_spaces.h>
 #include <vespa/storage/common/bucketoperationlogger.h>
 #include <vespa/storageapi/message/persistence.h>
 #include <vespa/storageapi/message/removelocation.h>
@@ -142,7 +143,8 @@ BucketDBUpdater::removeSuperfluousBuckets(
         if (!is_distribution_config_change
             && db_pruning_may_be_elided(oldClusterState, *new_cluster_state, up_states))
         {
-            LOG(debug, "Eliding DB pruning for state transition '%s' -> '%s'",
+            LOG(info, "[bucket space '%s']: eliding DB pruning for state transition '%s' -> '%s'",
+                document::FixedBucketSpaces::to_string(elem.first).data(),
                 oldClusterState.toString().c_str(), new_cluster_state->toString().c_str());
             continue;
         }