drop old summary fields in v8 mode

* the extra summary fields with .position and .distance will
  only be added in v7-geo-positions legacy mode on Vespa 8.

11 files changed, 72 insertions, 51 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index a492ee47030..62e78db0e6f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -74,6 +74,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private String mergeThrottlingPolicy = "STATIC";
     private double persistenceThrottlingWsDecrementFactor = 1.2;
     private double persistenceThrottlingWsBackoff = 0.95;
+    private boolean useV8GeoPositions = false;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -127,6 +128,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; }
     @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; }
     @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; }
+    @Override public boolean useV8GeoPositions() { return useV8GeoPositions; }
 
     public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) {
         this.maxUnCommittedMemory = maxUnCommittedMemory;
@@ -334,6 +336,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setUseV8GeoPositions(boolean value) {
+        this.useV8GeoPositions = value;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java
index 766b6ed3fec..254b3743f52 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.DataType;
 import com.yahoo.document.PositionDataType;
@@ -26,6 +27,14 @@ public class AdjustPositionSummaryFields extends Processor {
         super(schema, deployLogger, rankProfileRegistry, queryProfiles);
     }
 
+    private boolean useV8GeoPositions = false;
+
+    @Override
+    public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) {
+        this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions();
+        process(validate, documentsOnly);
+    }
+
     @Override
     public void process(boolean validate, boolean documentsOnly) {
         for (DocumentSummary summary : schema.getSummaries().values()) {
@@ -80,6 +89,7 @@ public class AdjustPositionSummaryFields extends Processor {
     private void ensureSummaryField(DocumentSummary summary, String fieldName, DataType dataType, Source source, SummaryTransform transform) {
         SummaryField oldField = schema.getSummaryField(fieldName);
         if (oldField == null) {
+            if (useV8GeoPositions) return;
             SummaryField newField = new SummaryField(fieldName, dataType, transform);
             newField.addSource(source);
             summary.add(newField);
@@ -94,6 +104,7 @@ public class AdjustPositionSummaryFields extends Processor {
         if (oldField.getSourceCount() != 1 || !oldField.getSingleSource().equals(source.getName())) {
             fail(oldField, "has source '" + oldField.getSources().toString() + "', should have source '" + source + "'");
         }
+        if (useV8GeoPositions) return;
         summary.add(oldField);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
index f5c1d8d8197..0bb1b7da769 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.DataType;
@@ -38,6 +39,14 @@ public class CreatePositionZCurve extends Processor {
         super(schema, deployLogger, rankProfileRegistry, queryProfiles);
     }
 
+    private boolean useV8GeoPositions = false;
+
+    @Override
+    public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) {
+        this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions();
+        process(validate, documentsOnly);
+    }
+
     @Override
     public void process(boolean validate, boolean documentsOnly) {
         for (SDField field : schema.allConcreteFields()) {
@@ -63,14 +72,16 @@ public class CreatePositionZCurve extends Processor {
 
             // configure summary
             Collection<String> summaryTo = removeSummaryTo(field);
-            ensureCompatibleSummary(field, zName,
-                                    PositionDataType.getPositionSummaryFieldName(fieldName),
-                                    DataType.getArray(DataType.STRING), // will become "xmlstring"
-                                    SummaryTransform.POSITIONS, summaryTo, validate);
-            ensureCompatibleSummary(field, zName,
-                                    PositionDataType.getDistanceSummaryFieldName(fieldName),
-                                    DataType.INT,
-                                    SummaryTransform.DISTANCE, summaryTo, validate);
+            if (! useV8GeoPositions) {
+                ensureCompatibleSummary(field, zName,
+                                        PositionDataType.getPositionSummaryFieldName(fieldName),
+                                        DataType.getArray(DataType.STRING), // will become "xmlstring"
+                                        SummaryTransform.POSITIONS, summaryTo, validate);
+                ensureCompatibleSummary(field, zName,
+                                        PositionDataType.getDistanceSummaryFieldName(fieldName),
+                                        DataType.INT,
+                                        SummaryTransform.DISTANCE, summaryTo, validate);
+            }
             // clear indexing script
             field.setIndexingScript(null);
             SDField posX = field.getStructField(PositionDataType.FIELD_X);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
index 9eb8b921e81..0db6f4f05ba 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
@@ -48,7 +48,7 @@ public class ImplicitSummaries extends Processor {
         sdField.addSummaryFieldSources(summaryField);
     }
 
-    private void collectSummaries(SDField field , Schema schema, boolean validate) {
+    private void collectSummaries(SDField field, Schema schema, boolean validate) {
         SummaryField addedSummaryField = null;
 
         // Implicit
@@ -91,9 +91,16 @@ public class ImplicitSummaries extends Processor {
         if (field.doesSummarying()) {
             for (Attribute attribute : field.getAttributes().values()) {
                 if ( ! attribute.isPosition()) continue;
-                DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema);
-                attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)));
-                attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)));
+                var distField = field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName));
+                if (distField != null) {
+                    DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema);
+                    attributePrefetchSummary.add(distField);
+                }
+                var posField = field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName));
+                if (posField != null) {
+                    DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema);
+                    attributePrefetchSummary.add(posField);
+                }
             }
         }
 
@@ -104,7 +111,6 @@ public class ImplicitSummaries extends Processor {
             if (attribute != null && summaryField.getTransform() == SummaryTransform.NONE) {
                 summaryField.setTransform(SummaryTransform.ATTRIBUTE);
             }
-
             if (isValid(summaryField, schema, validate)) {
                 addToDestinations(summaryField, schema);
             }
@@ -203,8 +209,9 @@ public class ImplicitSummaries extends Processor {
             addToDestination("default", summaryField, schema);
         }
         else {
-            for (String destinationName : summaryField.getDestinations())
+            for (String destinationName : summaryField.getDestinations()) {
                 addToDestination(destinationName, summaryField, schema);
+            }
         }
     }
 
diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg
index cd7c295ab11..2c46031bdad 100644
--- a/config-model/src/test/derived/position_nosummary/summary.cfg
+++ b/config-model/src/test/derived/position_nosummary/summary.cfg
@@ -1,12 +1,8 @@
-defaultsummaryid 1727020212
-usev8geopositions false
-classes[].id 1727020212
+defaultsummaryid 1151071433
+usev8geopositions true
+classes[].id 1151071433
 classes[].name "default"
 classes[].omitsummaryfeatures false
-classes[].fields[].name "pos.position"
-classes[].fields[].type "xmlstring"
-classes[].fields[].name "pos.distance"
-classes[].fields[].type "integer"
 classes[].fields[].name "rankfeatures"
 classes[].fields[].type "featuredata"
 classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/position_nosummary/summarymap.cfg b/config-model/src/test/derived/position_nosummary/summarymap.cfg
index cc1c14a6614..0a36f98c6ca 100644
--- a/config-model/src/test/derived/position_nosummary/summarymap.cfg
+++ b/config-model/src/test/derived/position_nosummary/summarymap.cfg
@@ -1,10 +1,4 @@
 defaultoutputclass -1
-override[].field "pos.position"
-override[].command "positions"
-override[].arguments "pos_zcurve"
-override[].field "pos.distance"
-override[].command "absdist"
-override[].arguments "pos_zcurve"
 override[].field "rankfeatures"
 override[].command "rankfeatures"
 override[].arguments ""
diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg
index 7b8bf16287f..7fda1ca0c05 100644
--- a/config-model/src/test/derived/position_summary/summary.cfg
+++ b/config-model/src/test/derived/position_summary/summary.cfg
@@ -1,14 +1,10 @@
-defaultsummaryid 230670304
-usev8geopositions false
-classes[].id 230670304
+defaultsummaryid 644855547
+usev8geopositions true
+classes[].id 644855547
 classes[].name "default"
 classes[].omitsummaryfeatures false
 classes[].fields[].name "pos"
 classes[].fields[].type "jsonstring"
-classes[].fields[].name "pos.position"
-classes[].fields[].type "xmlstring"
-classes[].fields[].name "pos.distance"
-classes[].fields[].type "integer"
 classes[].fields[].name "rankfeatures"
 classes[].fields[].type "featuredata"
 classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/position_summary/summarymap.cfg b/config-model/src/test/derived/position_summary/summarymap.cfg
index de48a19b6db..2223d3380a0 100644
--- a/config-model/src/test/derived/position_summary/summarymap.cfg
+++ b/config-model/src/test/derived/position_summary/summarymap.cfg
@@ -2,12 +2,6 @@ defaultoutputclass -1
 override[].field "pos"
 override[].command "geopos"
 override[].arguments "pos_zcurve"
-override[].field "pos.position"
-override[].command "positions"
-override[].arguments "pos_zcurve"
-override[].field "pos.distance"
-override[].command "absdist"
-override[].arguments "pos_zcurve"
 override[].field "rankfeatures"
 override[].command "rankfeatures"
 override[].arguments ""
diff --git a/config-model/src/test/derived/position_summary/vsmsummary.cfg b/config-model/src/test/derived/position_summary/vsmsummary.cfg
index d7d45782843..56405069131 100644
--- a/config-model/src/test/derived/position_summary/vsmsummary.cfg
+++ b/config-model/src/test/derived/position_summary/vsmsummary.cfg
@@ -2,12 +2,6 @@ outputclass ""
 fieldmap[].summary "pos"
 fieldmap[].document[].field "pos"
 fieldmap[].command NONE
-fieldmap[].summary "pos.position"
-fieldmap[].document[].field "pos_zcurve"
-fieldmap[].command NONE
-fieldmap[].summary "pos.distance"
-fieldmap[].document[].field "pos_zcurve"
-fieldmap[].command NONE
 fieldmap[].summary "rankfeatures"
 fieldmap[].command NONE
 fieldmap[].summary "summaryfeatures"
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
index a88ad929e80..456efdb08ae 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
@@ -100,6 +100,11 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase {
                                                          DeployLogger logger) throws IOException, ParseException {
         return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger);
     }
+    protected DerivedConfiguration assertCorrectDeriving(String dirName,
+                                                         TestProperties properties) throws IOException, ParseException
+    {
+        return assertCorrectDeriving(dirName, null, properties, new TestableDeployLogger());
+    }
 
     protected DerivedConfiguration assertCorrectDeriving(String dirName,
                                                          String searchDefinitionName,
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index ddb627216b7..27ba4e906a1 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.searchdefinition.ApplicationBuilder;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
@@ -23,27 +24,32 @@ public class ExportingTestCase extends AbstractExportingTestCase {
 
     @Test
     public void testPositionArray() throws IOException, ParseException {
-        assertCorrectDeriving("position_array");
+        assertCorrectDeriving("position_array",
+                              new TestProperties().setUseV8GeoPositions(true));
     }
 
     @Test
     public void testPositionAttribute() throws IOException, ParseException {
-        assertCorrectDeriving("position_attribute");
+        assertCorrectDeriving("position_attribute",
+                              new TestProperties().setUseV8GeoPositions(true));
     }
 
     @Test
     public void testPositionExtra() throws IOException, ParseException {
-        assertCorrectDeriving("position_extra");
+        assertCorrectDeriving("position_extra",
+                              new TestProperties().setUseV8GeoPositions(true));
     }
 
     @Test
     public void testPositionNoSummary() throws IOException, ParseException {
-        assertCorrectDeriving("position_nosummary");
+        assertCorrectDeriving("position_nosummary",
+                              new TestProperties().setUseV8GeoPositions(true));
     }
 
     @Test
     public void testPositionSummary() throws IOException, ParseException {
-        assertCorrectDeriving("position_summary");
+        assertCorrectDeriving("position_summary",
+                              new TestProperties().setUseV8GeoPositions(true));
     }
 
     @Test