diff options
7 files changed, 114 insertions, 33 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java index df189389348..1b2534e4cae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DocumentType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.Field; import com.yahoo.searchdefinition.Search; @@ -15,7 +16,6 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; */ public class BuiltInFieldSets extends Processor { - private static final String DOC_FIELDSET_NAME = "[document]"; public static final String SEARCH_FIELDSET_NAME = "[search]"; // Public due to oddities in position handling. public static final String INTERNAL_FIELDSET_NAME = "[internal]"; // This one populated from misc places @@ -43,7 +43,7 @@ public class BuiltInFieldSets extends Processor { if (docField instanceof SDField && ((SDField) docField).isExtraField()) { continue; // skip } - search.fieldSets().addBuiltInFieldSetItem(DOC_FIELDSET_NAME, docField.getName()); + search.fieldSets().addBuiltInFieldSetItem(DocumentType.DOCUMENT, docField.getName()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java index 8e221a6210b..13c967c8355 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; @@ -31,7 +32,7 @@ public class PositionTestCase { "src/test/examples/position_inherited.sd")); Search search = sb.getSearch("position_inherited"); - FieldSet fieldSet = search.getDocument().getFieldSets().builtInFieldSets().get("[document]"); // TODO why is this not public in BuiltInFieldSets? + FieldSet fieldSet = search.getDocument().getFieldSets().builtInFieldSets().get(DocumentType.DOCUMENT); assertFalse(fieldSet.getFieldNames().contains(PositionDataType.getZCurveFieldName("pos"))); } diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java index 17bcfd68bf4..5893013a187 100644 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java @@ -5,11 +5,17 @@ import com.yahoo.collections.Pair; import com.yahoo.docproc.CallStack; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.Processing; -import com.yahoo.document.*; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.update.FieldUpdate; import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; @@ -19,14 +25,17 @@ import com.yahoo.messagebus.Reply; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; /** * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> @@ -41,6 +50,7 @@ public class DocumentProcessingHandlerAllMessageTypesTestCase extends DocumentPr this.type.addField(new Field("blahblah", DataType.STRING)); this.type.addField(new Field("defaultWait", DataType.INT)); this.type.addField(new Field("customWait", DataType.INT)); + type.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("blahblah", "defaultWait", "customWait"))); } @Test diff --git a/document/abi-spec.json b/document/abi-spec.json index d6ee31695da..6ce2543b4c2 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -437,6 +437,7 @@ "protected void register(com.yahoo.document.DocumentTypeManager, java.util.List)", "public boolean isA(java.lang.String)", "public void addField(com.yahoo.document.Field)", + "public void addFieldSets(java.util.Map)", "public com.yahoo.document.Field addField(java.lang.String, com.yahoo.document.DataType)", "public com.yahoo.document.Field addHeaderField(java.lang.String, com.yahoo.document.DataType)", "public void inherit(com.yahoo.document.DocumentType)", @@ -450,6 +451,8 @@ "public com.yahoo.document.Field removeField(java.lang.String)", "public java.util.Collection getFields()", "public java.util.Set fieldSet()", + "public java.util.Set fieldSetAll()", + "public java.util.Set fieldSet(java.lang.String)", "public java.util.Iterator fieldIteratorThisTypeOnly()", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -462,6 +465,7 @@ "public bridge synthetic java.lang.Object clone()" ], "fields": [ + "public static final java.lang.String DOCUMENT", "public static final int classId" ] }, diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index 3a99c216c3f..2c9fe7b30d1 100755 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -12,8 +12,11 @@ import com.yahoo.vespa.objects.Serializer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.ListIterator; @@ -32,10 +35,13 @@ import java.util.Set; // TODO: Remove header/body concept on Vespa 8 public class DocumentType extends StructuredDataType { + private static final String ALL = "[all]"; + public static final String DOCUMENT = "[document]"; public static final int classId = registerClass(Ids.document + 58, DocumentType.class); private StructDataType headerType; private StructDataType bodyType; private List<DocumentType> inherits = new ArrayList<>(1); + private Map<String, Set<Field>> fieldSets = new HashMap<>(); /** * Creates a new document type and registers it with the document type manager. @@ -135,7 +141,7 @@ public class DocumentType extends StructuredDataType { header.clearFields(); body.clearFields(); - for (Field field : fieldSet()) { + for (Field field : getAllUniqueFields()) { (field.isHeader() ? header : body).addField(field); } headerType.assign(header); @@ -180,6 +186,30 @@ public class DocumentType extends StructuredDataType { struct.addField(field); } + // Do not use, public only for testing + public void addFieldSets(Map<String, Collection<String>> fieldSets) { + for (Map.Entry<String, Collection<String>> entry : fieldSets.entrySet()) { + + Set<Field> fields = new LinkedHashSet<>(entry.getValue().size()); + for (DocumentType parent : inherits) { + Set<Field> parentFieldSet = parent.fieldSet(entry.getKey()); + if (parentFieldSet != null) { + fields.addAll(parentFieldSet); + } + } + for (Field orderedField : getAllUniqueFields()) { + if (entry.getValue().contains(orderedField.getName())) { + fields.add(orderedField); + } + } + + this.fieldSets.put(entry.getKey(), ImmutableSet.copyOf(fields)); + } + if ( ! this.fieldSets.containsKey(ALL)) { + this.fieldSets.put(ALL, getAllUniqueFields()); + } + } + /** * Adds a new body field to this document type and returns the new field object * @@ -249,7 +279,7 @@ public class DocumentType extends StructuredDataType { * TODO Add strict type checking no duplicate fields are allowed */ private void verifyTypeConsistency(DocumentType superType) { - for (Field f : fieldSet()) { + for (Field f : getAllUniqueFields()) { Field supField = superType.getField(f.getName()); if (supField != null) { if (!f.getDataType().equals(supField.getDataType())) { @@ -341,9 +371,6 @@ public class DocumentType extends StructuredDataType { return getField(name) != null; } - //@Override - - public int getFieldCount() { return headerType.getFieldCount() + bodyType.getFieldCount(); } @@ -383,6 +410,14 @@ public class DocumentType extends StructuredDataType { return ImmutableList.copyOf(collection); } + private Set<Field> getAllUniqueFields() { + Map<String, Field> map = new LinkedHashMap<>(); + for (Field field : getFields()) { // Uniqify on field name + map.put(field.getName(), field); + } + return ImmutableSet.copyOf(map.values()); + } + /** * <p>Returns an ordered set snapshot of all fields of this documenttype, * <i>except the fields of Document</i>. @@ -400,11 +435,19 @@ public class DocumentType extends StructuredDataType { * @return an unmodifiable snapshot of the fields in this type */ public Set<Field> fieldSet() { - Map<String, Field> map = new LinkedHashMap<>(); - for (Field field : getFields()) { // Uniqify on field name - map.put(field.getName(), field); - } - return ImmutableSet.copyOf(map.values()); + return fieldSet(DOCUMENT); + } + + /** + * This is identical to @link fieldSet, but in addition extra hidden synthetic fields are returned. + * @return an unmodifiable snapshot of the all fields in this type + */ + public Set<Field> fieldSetAll() { + return fieldSet(ALL); + } + + public Set<Field> fieldSet(String name) { + return fieldSets.get(name); } /** diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index be5c6856ee5..154c25880a9 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -8,6 +8,9 @@ import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.log.LogLevel; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.logging.Logger; /** @@ -152,6 +155,11 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } type.inherit(parentType); } + Map<String, Collection<String>> fieldSets = new HashMap<>(doc.fieldsets().size()); + for (Map.Entry<String, DocumentmanagerConfig.Datatype.Documenttype.Fieldsets> entry: doc.fieldsets().entrySet()) { + fieldSets.put(entry.getKey(), entry.getValue().fields()); + } + type.addFieldSets(fieldSets); manager.register(type); } diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java index abbfd31b778..47c7fe71343 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java @@ -3,10 +3,13 @@ package com.yahoo.document; import org.junit.Test; +import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -29,35 +32,49 @@ public class DocumentTypeTestCase { } @Test + public void testFieldSets() { + DocumentType root = new DocumentType("root"); + root.addField("rootfield1", DataType.STRING); + root.addField("rootfield2", DataType.STRING); + root.addField("rootfield3", DataType.STRING); + root.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("rootfield2"))); + assertEquals(1, root.fieldSet().size()); + assertEquals("rootfield2", root.fieldSet().iterator().next().getName()); + assertEquals(3, root.fieldSetAll().size()); + } + + @Test public void testInheritance() { DocumentTypeManager typeManager = new DocumentTypeManager(); - DocumentType child = new DocumentType("child"); - Iterator inherited; - - child.addField("childfield", DataType.INT); - child.addField("overridden", DataType.STRING); + DocumentType root = new DocumentType("root"); + root.addField("rootfield", DataType.STRING); + root.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("rootfield"))); DocumentType parent1 = new DocumentType("parent1"); parent1.addField("overridden", DataType.STRING); parent1.addField("parent1field", DataType.STRING); - child.inherit(parent1); + parent1.inherit(root); + parent1.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("parent1field", "overridden"))); DocumentType parent2 = new DocumentType("parent2"); parent2.addField("parent2field", DataType.STRING); - child.inherit(parent2); - - DocumentType root = new DocumentType("root"); - root.addField("rootfield", DataType.STRING); - parent1.inherit(root); parent2.inherit(root); + parent2.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("parent2field"))); + + DocumentType child = new DocumentType("child"); + child.addField("childfield", DataType.INT); + child.addField("overridden", DataType.STRING); + child.inherit(parent1); + child.inherit(parent2); + child.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("childfield", "overridden"))); typeManager.register(root); typeManager.register(parent1); typeManager.register(parent2); typeManager.register(child); - inherited = child.getInheritedTypes().iterator(); + Iterator inherited = child.getInheritedTypes().iterator(); assertEquals(parent1, inherited.next()); assertEquals(parent2, inherited.next()); assertTrue(!inherited.hasNext()); @@ -95,9 +112,7 @@ public class DocumentTypeTestCase { assertFalse(fields.hasNext()); - assert(child.getField("rootfield") != null); - - // TODO: Test uninheriting + assertNotNull(child.getField("rootfield")); } } |