configurable rendering of "position" structs

8 files changed, 236 insertions, 8 deletions

diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index e43ff26272a..73ee2ecaedd 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -3,9 +3,10 @@ package com.yahoo.document;
 
 import com.yahoo.compress.CompressionType;
 import com.yahoo.config.subscription.ConfigSubscriber;
-import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.document.annotation.AnnotationReferenceDataType;
 import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.document.internal.GeoPosType;
 import java.util.logging.Level;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -97,11 +98,19 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
             }
         }
 
+        boolean looksLikePosition(StructDataType type) {
+            var pos = PositionDataType.INSTANCE;
+            return type.getName().equals(pos.getName()) && type.getId() == pos.getId();
+        }
+
         private void startStructsAndDocs(DocumentmanagerConfig config) {
             for (var thisDataType : config.datatype()) {
                 for (var o : thisDataType.structtype()) {
                     int id = thisDataType.id();
                     StructDataType type = new StructDataType(id, o.name());
+                    if (usev8geopositions && looksLikePosition(type)) {
+                        type = new GeoPosType(8);
+                    }
                     inProgress(type);
                     configMap.remove(id);
                 }
@@ -198,6 +207,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                 for (var struct : thisDataType.structtype()) {
                     int id = thisDataType.id();
                     StructDataType type = (StructDataType) typesById.get(id);
+                    if (type instanceof GeoPosType) {
+                        continue;
+                    }
                     for (var parent : struct.inherits()) {
                         var parentStruct = (StructDataType) typesByName.get(parent.name());
                         type.inherit(parentStruct);
@@ -319,9 +331,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         private final DocumentTypeManager manager;
     }
 
-
     private static class ApplyNewDoctypeConfig {
 
+
         public ApplyNewDoctypeConfig(DocumentmanagerConfig config, DocumentTypeManager manager) {
             this.manager = manager;
             this.usev8geopositions = config.usev8geopositions();
@@ -379,7 +391,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                 for (var typeconf : docTypeConfig.primitivetype()) {
                     DataType type = manager.getDataType(typeconf.name());
                     if (! (type instanceof PrimitiveDataType)) {
-                        throw new IllegalArgumentException("Needed primitive type for idx "+typeconf.idx()+" but got: "+type);
+                        throw new IllegalArgumentException("Needed primitive type for '"+typeconf.name()+"' [idx "+typeconf.idx()+"] but got: "+type);
                     }
                     addNewType(typeconf.idx(), type);
                 }
@@ -411,10 +423,32 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                 }
             }
 
+            private final Field POS_X = PositionDataType.INSTANCE.getField(PositionDataType.FIELD_X);
+            private final Field POS_Y = PositionDataType.INSTANCE.getField(PositionDataType.FIELD_Y);
+
+            boolean isPositionStruct(DocumentmanagerConfig.Doctype.Structtype cfg) {
+                if (! cfg.name().equals(PositionDataType.STRUCT_NAME)) return false;
+                if (! cfg.inherits().isEmpty()) return false;
+                if (cfg.field().size() != 2) return false;
+                var f0 = cfg.field(0);
+                var f1 = cfg.field(1);
+                if (! f0.name().equals(POS_X.getName())) return false;
+                if (! f1.name().equals(POS_Y.getName())) return false;
+                if (f0.internalid() != POS_X.getId()) return false;
+                if (f1.internalid() != POS_Y.getId()) return false;
+                if (typesByIdx.get(f0.type()) != POS_X.getDataType()) return false;
+                if (typesByIdx.get(f1.type()) != POS_Y.getDataType()) return false;
+                return true;
+            }
+
             void createEmptyStructs() {
                 String docName = docTypeConfig.name();
                 for (var typeconf : docTypeConfig.structtype()) {
-                    addNewType(typeconf.idx(), new StructDataType(typeconf.name()));
+                    if (usev8geopositions && isPositionStruct(typeconf)) {
+                        addNewType(typeconf.idx(), new GeoPosType(8));
+                    } else {
+                        addNewType(typeconf.idx(), new StructDataType(typeconf.name()));
+                    }
                 }
             }
 
@@ -486,6 +520,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
             }
             void fillStructs() {
                 for (var structCfg : docTypeConfig.structtype()) {
+                    if (usev8geopositions && isPositionStruct(structCfg)) {
+                        continue;
+                    }
                     int idx = structCfg.idx();
                     StructDataType type = (StructDataType) typesByIdx.get(idx);
                     for (var parent : structCfg.inherits()) {
@@ -541,11 +578,11 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
             }
             for (var docType : config.doctype()) {
                 var docTypeData = inProgressById.get(docType.idx());
+                docTypeData.createSimpleTypes();
                 docTypeData.createEmptyStructs();
                 docTypeData.initializeDocType();
                 docTypeData.createEmptyAnnotationTypes();
                 docTypeData.createFactories();
-                docTypeData.createSimpleTypes();
             }
             createComplexTypes();
             for (var docType : config.doctype()) {
diff --git a/document/src/main/java/com/yahoo/document/internal/GeoPosType.java b/document/src/main/java/com/yahoo/document/internal/GeoPosType.java
new file mode 100644
index 00000000000..2999f7506ee
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/internal/GeoPosType.java
@@ -0,0 +1,74 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.document.internal;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.Struct;
+
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.Locale;
+
+/**
+ * @author arnej
+ **/
+public final class GeoPosType extends StructDataType {
+
+    private final boolean useV8json;
+    private static final Field F_X = new Field("x", DataType.INT);
+    private static final Field F_Y = new Field("y", DataType.INT);
+
+    public GeoPosType(int vespaVersion) {
+        super("position");
+        this.useV8json = (vespaVersion == 8);
+        assert(vespaVersion > 6);
+        assert(vespaVersion < 9);
+        addField(F_X);
+        addField(F_Y);
+    }
+
+    public boolean renderJsonAsVespa8() {
+        return this.useV8json;
+    }
+
+    public double getLatitude(Struct pos) {
+        assert(pos.getDataType() == this);
+        double ns = PositionDataType.getYValue(pos).getInteger() * 1.0e-6;
+        return ns;
+    }
+
+    public double getLongitude(Struct pos) {
+        assert(pos.getDataType() == this);
+        double ew = PositionDataType.getXValue(pos).getInteger() * 1.0e-6;
+        return ew;
+    }
+
+    private static final DecimalFormat degreeFmt;
+
+    static {
+        degreeFmt = new DecimalFormat("0.0#####", DecimalFormatSymbols.getInstance(Locale.ENGLISH));
+        degreeFmt.setMinimumIntegerDigits(1);
+        degreeFmt.setMinimumFractionDigits(1);
+        degreeFmt.setMaximumFractionDigits(6);
+    }
+
+    static String fmtD(double degrees) {
+        return degreeFmt.format(degrees);
+    }
+
+    public String fmtLatitude(Struct pos) {
+        assert(pos.getDataType() == this);
+        double ns = PositionDataType.getYValue(pos).getInteger() * 1.0e-6;
+        return fmtD(ns);
+    }
+
+    public String fmtLongitude(Struct pos) {
+        assert(pos.getDataType() == this);
+        double ew = PositionDataType.getXValue(pos).getInteger() * 1.0e-6;
+        return fmtD(ew);
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
index 1d9fd3aa1ec..340bd542885 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
@@ -25,6 +25,7 @@ import com.yahoo.document.datatypes.Struct;
 import com.yahoo.document.datatypes.StructuredFieldValue;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.internal.GeoPosType;
 import com.yahoo.document.json.readers.TensorReader;
 import com.yahoo.document.json.readers.TensorRemoveUpdateReader;
 import com.yahoo.document.serialization.FieldWriter;
@@ -153,12 +154,32 @@ public class JsonSerializationHelper {
         });
     }
 
+    private static void serializeGeoPos(JsonGenerator generator, FieldBase field, Struct value, GeoPosType dataType) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> {
+                generator.writeStartObject();
+                generator.writeFieldName("lat");
+                generator.writeRawValue(dataType.fmtLatitude(value));
+                generator.writeFieldName("lng");
+                generator.writeRawValue(dataType.fmtLongitude(value));
+                generator.writeEndObject();
+        });
+    }
+
     public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) {
-        if (value.getDataType() == PositionDataType.INSTANCE) {
+        DataType dt = value.getDataType();
+        // TODO remove in Vespa 8:
+        if (dt == PositionDataType.INSTANCE) {
             serializeString(generator, field, PositionDataType.renderAsString(value));
             return;
         }
-
+        if (dt instanceof GeoPosType) {
+            var gpt = (GeoPosType)dt;
+            if (gpt.renderJsonAsVespa8()) {
+                serializeGeoPos(generator, field, value, gpt);
+                return;
+            }
+        }
         serializeStructuredField(fieldWriter, generator, field, value);
     }
 
diff --git a/document/src/test/document/documentmanager.cfg b/document/src/test/document/documentmanager.cfg
index 6ceda63e606..a4cf62db0c7 100644
--- a/document/src/test/document/documentmanager.cfg
+++ b/document/src/test/document/documentmanager.cfg
@@ -1,3 +1,4 @@
+usev8geopositions true
 doctype[4]
 doctype[0].name "document"
 doctype[0].idx 1000
@@ -45,6 +46,14 @@ doctype[0].annotationtype[8].internalid 6
 doctype[0].annotationtype[8].datatype 1004
 doctype[0].structtype[0].idx 1001
 doctype[0].structtype[0].name document.header
+doctype[0].structtype[1].idx 10010
+doctype[0].structtype[1].name "position"
+doctype[0].structtype[1].field[0].name "x"
+doctype[0].structtype[1].field[0].internalid 914677694
+doctype[0].structtype[1].field[0].type 1002
+doctype[0].structtype[1].field[1].name "y"
+doctype[0].structtype[1].field[1].internalid 900009410
+doctype[0].structtype[1].field[1].type 1002
 doctype[1].name "foobar"
 doctype[1].idx 1014
 doctype[1].inherits[0].idx 1000
diff --git a/document/src/test/document/documentmanager.testv8pos.cfg b/document/src/test/document/documentmanager.testv8pos.cfg
new file mode 100644
index 00000000000..3f776748b79
--- /dev/null
+++ b/document/src/test/document/documentmanager.testv8pos.cfg
@@ -0,0 +1,31 @@
+usev8geopositions true
+doctype[2]
+doctype[0].name "document"
+doctype[0].idx 1000
+doctype[0].contentstruct 1001
+doctype[0].primitivetype[0].idx 1002
+doctype[0].primitivetype[0].name "int"
+doctype[0].structtype[0].idx 1001
+doctype[0].structtype[0].name document.header
+doctype[0].structtype[1].idx 10010
+doctype[0].structtype[1].name "position"
+doctype[0].structtype[1].field[0].name "x"
+doctype[0].structtype[1].field[0].internalid 914677694
+doctype[0].structtype[1].field[0].type 1002
+doctype[0].structtype[1].field[1].name "y"
+doctype[0].structtype[1].field[1].internalid 900009410
+doctype[0].structtype[1].field[1].type 1002
+doctype[1].name "foobar"
+doctype[1].idx 1014
+doctype[1].contentstruct 1015
+doctype[1].inherits[0].idx 1000
+doctype[1].arraytype[0].idx 1017
+doctype[1].arraytype[0].elementtype 10010
+doctype[1].structtype[0].idx 1015
+doctype[1].structtype[0].name foobar.header
+doctype[1].structtype[0].field[0].name "simplepos"
+doctype[1].structtype[0].field[0].internalid 1707020592
+doctype[1].structtype[0].field[0].type 10010
+doctype[1].structtype[0].field[1].name "arraypos"
+doctype[1].structtype[0].field[1].internalid 1055920092
+doctype[1].structtype[0].field[1].type 1017
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
index 0aa5aec4b85..b89ed2b6b08 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
@@ -114,7 +114,7 @@ public class DocumentTypeManagerTestCase {
         assertSame(docType2, manager.getDocumentType(new DataTypeName("foo1")));
         assertSame(docType3, manager.getDocumentType(new DataTypeName("foo2")));
         assertSame(docType4, manager.getDocumentType(new DataTypeName("foo3")));
-        
+
         assertEquals(manager.getDocumentTypes().size(), 5);
         assertNotNull(manager.getDocumentTypes().get(new DataTypeName("document")));
         assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo0")), docType1);
@@ -587,6 +587,18 @@ search annotationsimplicitstruct {
         assertFalse(docType.hasImportedField("a_missing_imported_field"));
     }
 
+    @Test
+    public void position_type_is_recognized_as_v8() {
+        var manager = DocumentTypeManager.fromFile("src/test/document/documentmanager.testv8pos.cfg");
+        var docType = manager.getDocumentType("foobar");
+        var simplepos = docType.getField("simplepos").getDataType();
+        assertTrue(simplepos instanceof StructDataType);
+        var arraypos = docType.getField("arraypos").getDataType();
+        assertTrue(arraypos instanceof ArrayDataType);
+        var array = (ArrayDataType) arraypos;
+        assertTrue(array.getNestedType() instanceof StructDataType);
+    }
+
     // TODO test clone(). Also fieldSets not part of clone()..!
 
     // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this...
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 66ff7a7d4cd..ab4af5e722e 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -31,6 +31,7 @@ import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.Struct;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.internal.GeoPosType;
 import com.yahoo.document.json.readers.DocumentParseInfo;
 import com.yahoo.document.json.readers.VespaJsonDocumentReader;
 import com.yahoo.document.serialization.DocumentSerializer;
@@ -149,6 +150,7 @@ public class JsonReaderTestCase {
             DocumentType x = new DocumentType("testsinglepos");
             DataType d = PositionDataType.INSTANCE;
             x.addField(new Field("singlepos", d));
+            x.addField(new Field("geopos", new GeoPosType(8)));
             types.registerDocumentType(x);
         }
         {
@@ -612,6 +614,43 @@ public class JsonReaderTestCase {
     }
 
     @Test
+    public void testPositionGeoPos() throws IOException {
+        Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',",
+                "  'fields': {",
+                "    'geopos': 'N63.429722;E10.393333' }}"));
+        FieldValue f = doc.getFieldValue(doc.getField("geopos"));
+        assertSame(Struct.class, f.getClass());
+        assertEquals(10393333, PositionDataType.getXValue(f).getInteger());
+        assertEquals(63429722, PositionDataType.getYValue(f).getInteger());
+        assertEquals(f.getDataType(), PositionDataType.INSTANCE);
+    }
+
+    @Test
+    public void testPositionOldGeoPos() throws IOException {
+        Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',",
+                "  'fields': {",
+                "    'geopos': {'x':10393333,'y':63429722} }}"));
+        FieldValue f = doc.getFieldValue(doc.getField("geopos"));
+        assertSame(Struct.class, f.getClass());
+        assertEquals(10393333, PositionDataType.getXValue(f).getInteger());
+        assertEquals(63429722, PositionDataType.getYValue(f).getInteger());
+        assertEquals(f.getDataType(), PositionDataType.INSTANCE);
+    }
+
+    @Test
+    public void testGeoPositionGeoPos() throws IOException {
+        Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',",
+                "  'fields': {",
+                "    'geopos': {'lat':63.429722,'lng':10.393333} }}"));
+        FieldValue f = doc.getFieldValue(doc.getField("geopos"));
+        assertSame(Struct.class, f.getClass());
+        assertEquals(10393333, PositionDataType.getXValue(f).getInteger());
+        assertEquals(63429722, PositionDataType.getYValue(f).getInteger());
+        assertEquals(f.getDataType(), PositionDataType.INSTANCE);
+        assertEquals(PositionDataType.INSTANCE, f.getDataType());
+    }
+
+    @Test
     public void testPositionNegative() throws IOException {
         Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',",
                 "  'fields': {",
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
index 29703eadfce..7573aba519f 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -22,6 +22,7 @@ import com.yahoo.document.TensorDataType;
 import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.document.datatypes.ReferenceFieldValue;
 import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.internal.GeoPosType;
 import com.yahoo.document.json.readers.DocumentParseInfo;
 import com.yahoo.document.json.readers.VespaJsonDocumentReader;
 import com.yahoo.tensor.TensorType;
@@ -93,6 +94,7 @@ public class JsonWriterTestCase {
         DocumentType x = new DocumentType("testmultipos");
         DataType d = new ArrayDataType(PositionDataType.INSTANCE);
         x.addField(new Field("multipos", d));
+        x.addField(new Field("geopos", new ArrayDataType(new GeoPosType(8))));
         types.registerDocumentType(x);
     }
 
@@ -100,6 +102,7 @@ public class JsonWriterTestCase {
         DocumentType x = new DocumentType("testsinglepos");
         DataType d = PositionDataType.INSTANCE;
         x.addField(new Field("singlepos", d));
+        x.addField(new Field("geopos", new GeoPosType(8)));
         types.registerDocumentType(x);
     }
 
@@ -202,11 +205,13 @@ public class JsonWriterTestCase {
     @Test
     public void singlePosTest() throws IOException {
         roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"singlepos\": \"N60.222333;E10.12\" }");
+        roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"geopos\": { \"lat\": 60.222333, \"lng\": 10.12 } }");
     }
 
     @Test
     public void multiPosTest() throws IOException {
         roundTripEquality("id:unittest:testmultipos::bamf", "{ \"multipos\": [ \"N0.0;E0.0\", \"S1.1;W1.1\", \"N10.2;W122.2\" ] }");
+        roundTripEquality("id:unittest:testmultipos::bamf", "{ \"geopos\": [ {  \"lat\": -1.5, \"lng\": -1.5 }, {  \"lat\": 63.4, \"lng\": 10.4 }, { \"lat\": 0.0, \"lng\": 0.0 } ] }");
     }
 
     @Test