diff options
Diffstat (limited to 'config-model')
18 files changed, 621 insertions, 54 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java index 24a8d81c754..656f78ba2a9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java @@ -80,9 +80,8 @@ public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends Abst private static String getIdString(Element spec) { String idString = XmlHelper.getIdString(spec); - if (idString == null || idString.isEmpty()) { + if (idString.isEmpty()) idString = spec.getTagName(); - } return idString; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 6960a0a8afd..26b4b78fcaa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -965,12 +965,13 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> compiledFunctions = new LinkedHashMap<>(); Map.Entry<String, RankingExpressionFunction> entry; // Compile all functions. Why iterate in such a complicated way? - // Because some functions (imported models adding generated macros) may add other functions during compiling. + // Because some functions (imported models adding generated functions) may add other functions during compiling. // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes, - importedModels, getConstants(), inlineFunctions, expressionTransforms); + importedModels, getConstants(), inlineFunctions, + expressionTransforms); compiledFunctions.put(entry.getKey(), compiled); } return compiledFunctions; @@ -986,12 +987,12 @@ public class RankProfile implements Cloneable { } private RankingExpressionFunction compile(RankingExpressionFunction function, - QueryProfileRegistry queryProfiles, - Map<Reference, TensorType> featureTypes, - ImportedMlModels importedModels, - Map<String, Value> constants, - Map<String, RankingExpressionFunction> inlineFunctions, - ExpressionTransforms expressionTransforms) { + QueryProfileRegistry queryProfiles, + Map<Reference, TensorType> featureTypes, + ImportedMlModels importedModels, + Map<String, Value> constants, + Map<String, RankingExpressionFunction> inlineFunctions, + ExpressionTransforms expressionTransforms) { if (function == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java new file mode 100644 index 00000000000..9b69a82a8ff --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -0,0 +1,292 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Class converting a collection of schemas from the intermediate format. + * For now only conversion to DocumentType (with contents). + * + * @author arnej27959 + **/ +public class ConvertSchemaCollection { + + private final IntermediateCollection input; + private final List<ParsedSchema> orderedInput = new ArrayList<>(); + private final DocumentTypeManager docMan; + + public ConvertSchemaCollection(IntermediateCollection input, + DocumentTypeManager documentTypeManager) + { + this.input = input; + this.docMan = documentTypeManager; + order(); + pushTypesToDocuments(); + } + + public void convertTypes() { + convertDataTypes(); + registerDataTypes(); + } + + void order() { + var map = input.getParsedSchemas(); + for (var schema : map.values()) { + findOrdering(schema); + } + } + + void findOrdering(ParsedSchema schema) { + if (orderedInput.contains(schema)) return; + for (var parent : schema.getAllResolvedInherits()) { + findOrdering(parent); + } + orderedInput.add(schema); + } + + void pushTypesToDocuments() { + for (var schema : orderedInput) { + for (var struct : schema.getStructs()) { + schema.getDocument().addStruct(struct); + } + for (var annotation : schema.getAnnotations()) { + schema.getDocument().addAnnotation(annotation); + } + } + } + + Map<String, DocumentType> documentsInProgress = new HashMap<>(); + Map<String, StructDataType> structsInProgress = new HashMap<>(); + Map<String, AnnotationType> annotationsInProgress = new HashMap<>(); + + StructDataType findStructInProgress(String name, ParsedDocument context) { + var resolved = findStructFrom(context, name); + if (resolved == null) { + throw new IllegalArgumentException("no struct named " + name + " in context " + context); + } + String structId = resolved.getOwner() + "->" + resolved.name(); + var struct = structsInProgress.get(structId); + assert(struct != null); + return struct; + } + + AnnotationType findAnnotationInProgress(String name, ParsedDocument context) { + var resolved = findAnnotationFrom(context, name); + String annotationId = resolved.getOwner() + "->" + resolved.name(); + var annotation = annotationsInProgress.get(annotationId); + if (annotation == null) { + throw new IllegalArgumentException("no annotation named " + name + " in context " + context); + } + return annotation; + } + + ParsedStruct findStructFrom(ParsedDocument doc, String name) { + ParsedStruct found = doc.getStruct(name); + if (found != null) return found; + for (var parent : doc.getResolvedInherits()) { + var fromParent = findStructFrom(parent, name); + if (fromParent == null) continue; + if (fromParent == found) continue; + if (found == null) { + found = fromParent; + } else { + throw new IllegalArgumentException("conflicting values for struct " + name + " in " +doc); + } + } + return found; + } + + ParsedAnnotation findAnnotationFrom(ParsedDocument doc, String name) { + ParsedAnnotation found = doc.getAnnotation(name); + if (found != null) return found; + for (var parent : doc.getResolvedInherits()) { + var fromParent = findAnnotationFrom(parent, name); + if (fromParent == null) continue; + if (fromParent == found) continue; + if (found == null) { + found = fromParent; + } else { + throw new IllegalArgumentException("conflicting values for annotation " + name + " in " +doc); + } + } + return found; + } + + private DataType createArray(ParsedType pType, ParsedDocument context) { + DataType nested = resolveType(pType.nestedType(), context); + return DataType.getArray(nested); + } + + private DataType createWset(ParsedType pType, ParsedDocument context) { + DataType nested = resolveType(pType.nestedType(), context); + boolean cine = pType.getCreateIfNonExistent(); + boolean riz = pType.getRemoveIfZero(); + return new WeightedSetDataType(nested, cine, riz); + } + + private DataType createMap(ParsedType pType, ParsedDocument context) { + DataType kt = resolveType(pType.mapKeyType(), context); + DataType vt = resolveType(pType.mapValueType(), context); + return DataType.getMap(kt, vt); + } + + private DocumentType findDocInProgress(String name) { + var dt = documentsInProgress.get(name); + if (dt == null) { + throw new IllegalArgumentException("missing document type for: " + name); + } + return dt; + } + + private DataType createAnnRef(ParsedType pType, ParsedDocument context) { + AnnotationType annotation = findAnnotationInProgress(pType.getNameOfReferencedAnnotation(), context); + return new AnnotationReferenceDataType(annotation); + } + + private DataType createDocRef(ParsedType pType) { + var ref = pType.getReferencedDocumentType(); + assert(ref.getVariant() == ParsedType.Variant.DOCUMENT); + return ReferenceDataType.createWithInferredId(findDocInProgress(ref.name())); + } + + DataType resolveType(ParsedType pType, ParsedDocument context) { + switch (pType.getVariant()) { + case NONE: return DataType.NONE; + case BUILTIN: return docMan.getDataType(pType.name()); + case POSITION: return PositionDataType.INSTANCE; + case ARRAY: return createArray(pType, context); + case WSET: return createWset(pType, context); + case MAP: return createMap(pType, context); + case TENSOR: return DataType.getTensor(pType.getTensorType()); + case DOC_REFERENCE: return createDocRef(pType); + case ANN_REFERENCE: return createAnnRef(pType, context); + case DOCUMENT: return findDocInProgress(pType.name()); + case STRUCT: return findStructInProgress(pType.name(), context); + case UNKNOWN: + // fallthrough + } + // unknown is probably struct, but could be document: + if (documentsInProgress.containsKey(pType.name())) { + pType.setVariant(ParsedType.Variant.DOCUMENT); + return findDocInProgress(pType.name()); + } + var struct = findStructInProgress(pType.name(), context); + pType.setVariant(ParsedType.Variant.STRUCT); + return struct; + } + + void convertDataTypes() { + for (var schema : orderedInput) { + String name = schema.getDocument().name(); + documentsInProgress.put(name, new DocumentType(name)); + } + for (var schema : orderedInput) { + var doc = schema.getDocument(); + for (var struct : doc.getStructs()) { + var dt = new StructDataType(struct.name()); + String structId = doc.name() + "->" + struct.name(); + structsInProgress.put(structId, dt); + } + for (var annotation : doc.getAnnotations()) { + String annId = doc.name() + "->" + annotation.name(); + var at = new AnnotationType(annotation.name()); + annotationsInProgress.put(annId, at); + var withStruct = annotation.getStruct(); + if (withStruct.isPresent()) { + var sn = withStruct.get().name(); + var dt = new StructDataType(sn); + String structId = doc.name() + "->" + sn; + structsInProgress.put(structId, dt); + } + } + } + for (var schema : orderedInput) { + var doc = schema.getDocument(); + for (var struct : doc.getStructs()) { + String structId = doc.name() + "->" + struct.name(); + var toFill = structsInProgress.get(structId); + for (String inherit : struct.getInherited()) { + var parent = findStructInProgress(inherit, doc); + toFill.inherit(parent); + } + for (ParsedField field : struct.getFields()) { + var t = resolveType(field.getType(), doc); + var f = new com.yahoo.document.Field(field.name(), t); + toFill.addField(f); + } + } + for (var annotation : doc.getAnnotations()) { + String annId = doc.name() + "->" + annotation.name(); + var at = annotationsInProgress.get(annId); + var withStruct = annotation.getStruct(); + if (withStruct.isPresent()) { + ParsedStruct struct = withStruct.get(); + String structId = doc.name() + "->" + struct.name(); + var toFill = structsInProgress.get(structId); + for (ParsedField field : struct.getFields()) { + var t = resolveType(field.getType(), doc); + var f = new com.yahoo.document.Field(field.name(), t); + toFill.addField(f); + } + at.setDataType(toFill); + } + for (String inherit : annotation.getInherited()) { + var parent = findAnnotationInProgress(inherit, doc); + at.inherit(parent); + } + } + + var docToFill = documentsInProgress.get(doc.name()); + Map<String, Collection<String>> fieldSets = new HashMap<>(); + List<String> inDocFields = new ArrayList<>(); + for (var docField : doc.getFields()) { + String name = docField.name(); + var t = resolveType(docField.getType(), doc); + var f = new com.yahoo.document.Field(name, t); + docToFill.addField(f); + inDocFields.add(name); + } + fieldSets.put("[document]", inDocFields); + for (var extraField : schema.getFields()) { + String name = extraField.name(); + var t = resolveType(extraField.getType(), doc); + var f = new com.yahoo.document.Field(name, t); + docToFill.addField(f); + } + for (var fieldset : schema.getFieldSets()) { + fieldSets.put(fieldset.name(), fieldset.getFieldNames()); + } + docToFill.addFieldSets(fieldSets); + for (String inherit : doc.getInherited()) { + docToFill.inherit(findDocInProgress(inherit)); + } + } + } + + void registerDataTypes() { + for (DataType t : structsInProgress.values()) { + docMan.register(t); + } + for (DocumentType t : documentsInProgress.values()) { + docMan.register(t); + } + for (AnnotationType t : annotationsInProgress.values()) { + docMan.getAnnotationTypeRegistry().register(t); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java index 488464ccd1f..edcbf85b5dc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java @@ -28,7 +28,7 @@ public class InheritanceResolver { String.join(" -> ", seen)); } seen.add(name); - for (ParsedSchema parent : schema.getResolvedInherits()) { + for (ParsedSchema parent : schema.getAllResolvedInherits()) { inheritanceCycleCheck(parent, seen); } seen.remove(name); @@ -57,11 +57,23 @@ public class InheritanceResolver { for (ParsedSchema schema : parsedSchemas.values()) { if (! schema.hasDocument()) { // TODO: is schema without a document even valid? - continue; + // could make sense for schemas with just rank-profile functions + // it makes life easier to behave as if there was en empty + // document block here. + var doc = new ParsedDocument(schema.name()); + for (String inherit : schema.getInherited()) { + doc.inherit(inherit); + } + schema.addDocument(doc); } ParsedDocument doc = schema.getDocument(); var old = parsedDocs.put(doc.name(), doc); - assert(old == null); + if (old != null) { + throw new IllegalArgumentException("duplicate document declaration for " + doc.name()); + } + for (String docInherit : doc.getInherited()) { + schema.inheritByDocument(docInherit); + } } for (ParsedDocument doc : parsedDocs.values()) { for (String inherit : doc.getInherited()) { @@ -72,6 +84,14 @@ public class InheritanceResolver { doc.resolveInherit(inherit, parentDoc); } } + for (ParsedSchema schema : parsedSchemas.values()) { + for (String inherit : schema.getInheritedByDocument()) { + var parent = parsedSchemas.get(inherit); + assert(parent.hasDocument()); + assert(parent.getDocument().name().equals(inherit)); + schema.resolveInheritByDocument(inherit, parent); + } + } } private void inheritanceCycleCheck(ParsedDocument document, List<String> seen) { @@ -98,8 +118,8 @@ public class InheritanceResolver { public void resolveInheritance() { resolveSchemaInheritance(); resolveDocumentInheritance(); - checkSchemaCycles(); checkDocumentCycles(); + checkSchemaCycles(); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java index a4e38795850..f22d370b1d8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java @@ -14,6 +14,7 @@ class ParsedAnnotation extends ParsedBlock { private ParsedStruct wrappedStruct = null; private final List<String> inherited = new ArrayList<>(); + private String ownedBy = null; ParsedAnnotation(String name) { super(name, "annotation"); @@ -21,7 +22,12 @@ class ParsedAnnotation extends ParsedBlock { public List<String> getInherited() { return List.copyOf(inherited); } public Optional<ParsedStruct> getStruct() { return Optional.ofNullable(wrappedStruct); } + public String getOwner() { return ownedBy; } void setStruct(ParsedStruct struct) { this.wrappedStruct = struct; } void inherit(String other) { inherited.add(other); } + void tagOwner(String owner) { + verifyThat(ownedBy == null, "already owned by", ownedBy); + this.ownedBy = owner; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java index 8cd64ef16f7..dd61124c3a7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java @@ -31,6 +31,8 @@ public class ParsedDocument extends ParsedBlock { List<ParsedDocument> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } List<ParsedField> getFields() { return List.copyOf(docFields.values()); } List<ParsedStruct> getStructs() { return List.copyOf(docStructs.values()); } + ParsedStruct getStruct(String name) { return docStructs.get(name); } + ParsedAnnotation getAnnotation(String name) { return docAnnotations.get(name); } void inherit(String other) { inherited.add(other); } @@ -44,12 +46,14 @@ public class ParsedDocument extends ParsedBlock { String sName = struct.name(); verifyThat(! docStructs.containsKey(sName), "already has struct", sName); docStructs.put(sName, struct); + struct.tagOwner(name()); } void addAnnotation(ParsedAnnotation annotation) { String annName = annotation.name(); verifyThat(! docAnnotations.containsKey(annName), "already has annotation", annName); docAnnotations.put(annName, annotation); + annotation.tagOwner(name()); } public String toString() { return "document " + name(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index bf448b31dd2..a0b238f1f43 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -36,7 +36,9 @@ public class ParsedSchema extends ParsedBlock { private final List<OnnxModel> onnxModels = new ArrayList<>(); private final List<RankingConstant> rankingConstants = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); + private final List<String> inheritedByDocument = new ArrayList<>(); private final Map<String, ParsedSchema> resolvedInherits = new HashMap(); + private final Map<String, ParsedSchema> allResolvedInherits = new HashMap(); private final Map<String, ParsedAnnotation> extraAnnotations = new HashMap<>(); private final Map<String, ParsedDocumentSummary> docSums = new HashMap<>(); private final Map<String, ParsedField> extraFields = new HashMap<>(); @@ -64,8 +66,10 @@ public class ParsedSchema extends ParsedBlock { List<ParsedStruct> getStructs() { return List.copyOf(extraStructs.values()); } List<RankingConstant> getRankingConstants() { return List.copyOf(rankingConstants); } List<String> getInherited() { return List.copyOf(inherited); } + List<String> getInheritedByDocument() { return List.copyOf(inheritedByDocument); } Map<String, ParsedRankProfile> getRankProfiles() { return Map.copyOf(rankProfiles); } List<ParsedSchema> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } + List<ParsedSchema> getAllResolvedInherits() { return List.copyOf(allResolvedInherits.values()); } void addAnnotation(ParsedAnnotation annotation) { String annName = annotation.name(); @@ -76,6 +80,8 @@ public class ParsedSchema extends ParsedBlock { void addDocument(ParsedDocument document) { verifyThat(myDocument == null, "already has", myDocument, "so cannot add", document); + verifyThat(name().equals(document.name()), + "schema " + name() + "can only contain document named " + name() + ", was: "+ document.name()); this.myDocument = document; } @@ -133,6 +139,8 @@ public class ParsedSchema extends ParsedBlock { void inherit(String other) { inherited.add(other); } + void inheritByDocument(String other) { inheritedByDocument.add(other); } + void setStemming(Stemming value) { verifyThat((defaultStemming == null) || (defaultStemming == value), "already has stemming", defaultStemming, "cannot also set", value); @@ -144,6 +152,16 @@ public class ParsedSchema extends ParsedBlock { verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); resolvedInherits.put(name, parsed); + var old = allResolvedInherits.put(name, parsed); + verifyThat(old == null || old == parsed, "conflicting resolveInherit for", name); + } + + void resolveInheritByDocument(String name, ParsedSchema parsed) { + verifyThat(inheritedByDocument.contains(name), + "resolveInheritByDocument for non-inherited name", name); + verifyThat(name.equals(parsed.name()), "resolveInheritByDocument name mismatch for", name); + var old = allResolvedInherits.put(name, parsed); + verifyThat(old == null || old == parsed, "conflicting resolveInherit for", name); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java index 67f2f137bc1..cc3b2425726 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java @@ -15,6 +15,7 @@ import java.util.Map; public class ParsedStruct extends ParsedBlock { private final List<String> inherited = new ArrayList<>(); private final Map<String, ParsedField> fields = new HashMap<>(); + private String ownedBy = null; public ParsedStruct(String name) { super(name, "struct"); @@ -22,6 +23,7 @@ public class ParsedStruct extends ParsedBlock { List<ParsedField> getFields() { return List.copyOf(fields.values()); } List<String> getInherited() { return List.copyOf(inherited); } + String getOwner() { return ownedBy; } void addField(ParsedField field) { String fieldName = field.name(); @@ -29,6 +31,15 @@ public class ParsedStruct extends ParsedBlock { fields.put(fieldName, field); } - void inherit(String other) { inherited.add(other); } + void inherit(String other) { + verifyThat(! name().equals(other), "cannot inherit from itself"); + inherited.add(other); + } + + void tagOwner(String document) { + verifyThat(ownedBy == null, "already owned by document "+ownedBy); + this.ownedBy = document; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index 9f02c5247ef..d3e85bc1b11 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -13,10 +13,9 @@ import com.yahoo.tensor.TensorType; class ParsedType { public enum Variant { NONE, - BOOL, BYTE, INT, LONG, - STRING, - FLOAT, DOUBLE, - URI, PREDICATE, TENSOR, + BUILTIN, + POSITION, + TENSOR, ARRAY, WSET, MAP, DOC_REFERENCE, ANN_REFERENCE, @@ -35,16 +34,16 @@ class ParsedType { private static Variant guessVariant(String name) { switch (name) { - case "bool": return Variant.BOOL; - case "byte": return Variant.BYTE; - case "int": return Variant.INT; - case "long": return Variant.LONG; - case "string": return Variant.STRING; - case "float": return Variant.FLOAT; - case "double": return Variant.DOUBLE; - case "uri": return Variant.URI; - case "predicate": return Variant.PREDICATE; - case "position": return Variant.STRUCT; + case "bool": return Variant.BUILTIN; + case "byte": return Variant.BUILTIN; + case "int": return Variant.BUILTIN; + case "long": return Variant.BUILTIN; + case "string": return Variant.BUILTIN; + case "float": return Variant.BUILTIN; + case "double": return Variant.BUILTIN; + case "uri": return Variant.BUILTIN; + case "predicate": return Variant.BUILTIN; + case "position": return Variant.POSITION; } return Variant.UNKNOWN; } @@ -53,20 +52,28 @@ class ParsedType { public Variant getVariant() { return variant; } public ParsedType mapKeyType() { assert(variant == Variant.MAP); return keyType; } public ParsedType mapValueType() { assert(variant == Variant.MAP); return valType; } - public ParsedType nestedType() { assert(variant == Variant.ARRAY || variant == Variant.WSET); return valType; } + public ParsedType nestedType() { assert(variant == Variant.ARRAY || variant == Variant.WSET); assert(valType != null); return valType; } public boolean getCreateIfNonExistent() { assert(variant == Variant.WSET); return this.createIfNonExistent; } public boolean getRemoveIfZero() { assert(variant == Variant.WSET); return this.removeIfZero; } public ParsedType getReferencedDocumentType() { assert(variant == Variant.DOC_REFERENCE); return valType; } public TensorType getTensorType() { assert(variant == Variant.TENSOR); return tensorType; } + public String getNameOfReferencedAnnotation() { + assert(variant == Variant.ANN_REFERENCE); + String prefix = "annotationreference<"; + int fromPos = prefix.length(); + int toPos = name.length() - 1; + return name.substring(fromPos, toPos); + } + private ParsedType(String name, Variant variant) { this(name, variant, null, null, null); } private ParsedType(String name, Variant variant, ParsedType vt) { - this(name, variant, vt, null, null); + this(name, variant, null, vt, null); } private ParsedType(String name, Variant variant, ParsedType kt, ParsedType vt) { - this(name, variant, vt, kt, null); + this(name, variant, kt, vt, null); } private ParsedType(String name, Variant variant, ParsedType kt, ParsedType vt, TensorType tType) { this.name = name; @@ -77,22 +84,28 @@ class ParsedType { } static ParsedType mapType(ParsedType kt, ParsedType vt) { + assert(kt != null); + assert(vt != null); String name = "map<" + kt.name() + "," + vt.name() + ">"; return new ParsedType(name, Variant.MAP, kt, vt); } static ParsedType arrayOf(ParsedType vt) { + assert(vt != null); return new ParsedType("array<" + vt.name() + ">", Variant.ARRAY, vt); } static ParsedType wsetOf(ParsedType vt) { + assert(vt != null); return new ParsedType("weightedset<" + vt.name() + ">", Variant.WSET, vt); } static ParsedType documentRef(ParsedType docType) { + assert(docType != null); return new ParsedType("reference<" + docType.name + ">", Variant.DOC_REFERENCE, docType); } static ParsedType annotationRef(String name) { return new ParsedType("annotationreference<" + name + ">", Variant.ANN_REFERENCE); } static ParsedType tensorType(TensorType tType) { + assert(tType != null); return new ParsedType(tType.toString(), Variant.TENSOR, null, null, tType); } static ParsedType fromName(String name) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index c88d225f527..17690c6ecab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -824,6 +824,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .dockerImageRepository(deployState.getWantedDockerImageRepo()) .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; + deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + + " nodes in " + cluster); Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified()), false, !deployState.getProperties().isBootstrap()); diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index cc03773f333..a8e77dead6e 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -427,16 +427,11 @@ void rootSchemaItem(ParsedSchema schema) : { } */ ParsedSchema rootDocument() : { - ParsedSchema schema = new ParsedSchema("<unnamed>"); + ParsedSchema schema = null; } { - ( (rootDocumentItem(schema) (<NL>)*)*<EOF> ) + ( (schema = rootDocumentItem(schema) (<NL>)*)*<EOF> ) { - if (schema.hasDocument()) { - ParsedDocument doc = schema.getDocument(); - schema = new ParsedSchema(doc.name()); - schema.addDocument(doc); - } return schema; } } @@ -446,9 +441,16 @@ ParsedSchema rootDocument() : * * @param schema the schema object to modify. */ -void rootDocumentItem(ParsedSchema schema) : { } +ParsedSchema rootDocumentItem(ParsedSchema schema) : +{ + ParsedDocument doc = null; +} { - ( namedDocument(schema) ) + ( doc = namedDocument() { + if (schema == null) schema = new ParsedSchema(doc.name()); + schema.addDocument(doc); + return schema; + } ) } /** @@ -483,10 +485,8 @@ void document(ParsedSchema schema) : /** * Consumes a document element, explicitly named - * - * @param schema the schema object to add content to */ -void namedDocument(ParsedSchema schema) : +ParsedDocument namedDocument() : { String name; ParsedDocument document; @@ -496,7 +496,7 @@ void namedDocument(ParsedSchema schema) : [ inheritsDocument(document) (<NL>)* ] <LBRACE> (<NL>)* (documentBody(document) (<NL>)*)* <RBRACE> ) { - schema.addDocument(document); + return document; } } diff --git a/config-model/src/test/converter/child.sd b/config-model/src/test/converter/child.sd new file mode 100644 index 00000000000..cdfc339ed59 --- /dev/null +++ b/config-model/src/test/converter/child.sd @@ -0,0 +1,23 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search child { + + document child inherits parent { + + field a type uri { + indexing: index | summary + } + + field r type redef { + indexing: summary + } + + field aaa type annotationreference<gpa> { } + + field modelref type reference<other> { } + + } + field outrarr type array<string> { + indexing: input a | to_array | summary + } + +} diff --git a/config-model/src/test/converter/grandparent.sd b/config-model/src/test/converter/grandparent.sd new file mode 100644 index 00000000000..603553f739d --- /dev/null +++ b/config-model/src/test/converter/grandparent.sd @@ -0,0 +1,32 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search grandparent { + + struct item { + field f1i type int {} + } + + struct gps { + field reftoa type annotationreference<gpa> {} + field someitems type array<item> {} + } + + document grandparent { + + field c type map<string, gps> { + indexing: index + } + + #field inrgp type redef { + #} + } + + annotation gpa { + field city type string {} + field zip type int {} + } + + #struct redef { + # field y type int {} + # field z type string {} + #} +} diff --git a/config-model/src/test/converter/other.sd b/config-model/src/test/converter/other.sd new file mode 100644 index 00000000000..6cf2a56c43a --- /dev/null +++ b/config-model/src/test/converter/other.sd @@ -0,0 +1,16 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search other { + + document other { + + field c type tensor(d[512]) { + indexing: attribute + } + + field d type tensor<float>(cat{},x[13]) { + indexing: attribute + } + + } + +} diff --git a/config-model/src/test/converter/parent.sd b/config-model/src/test/converter/parent.sd new file mode 100644 index 00000000000..f05edaef787 --- /dev/null +++ b/config-model/src/test/converter/parent.sd @@ -0,0 +1,29 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search parent { + + struct ps { + field wil type weightedset<string> {} + field after type array<long> {} + field psi type item {} + } + + document parent inherits grandparent { + + field b type string { + indexing: index | summary + } + + field bps type ps { + indexing: summary + } + + field location type array<position> { + indexing: attribute + } + } + + struct redef { + field x type int {} + field y type string {} + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java new file mode 100644 index 00000000000..264481cb3ec --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import static com.yahoo.config.model.test.TestUtil.joinLines; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; + +/** + * @author arnej + */ +public class ConvertIntermediateTestCase { + + @Test + public void can_convert_minimal_schema() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " }", + "}"); + var collection = new IntermediateCollection(); + ParsedSchema schema = collection.addSchemaFromString(input); + assertEquals("foo", schema.getDocument().name()); + collection.resolveInternalConnections(); + var docMan = new DocumentTypeManager(); + var converter = new ConvertSchemaCollection(collection, docMan); + converter.convertTypes(); + var dt = docMan.getDocumentType("foo"); + assertTrue(dt != null); + } + + @Test + public void can_convert_schema_files() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); + assertEquals(collection.getParsedSchemas().size(), 3); + collection.resolveInternalConnections(); + var docMan = new DocumentTypeManager(); + var converter = new ConvertSchemaCollection(collection, docMan); + converter.convertTypes(); + var dt = docMan.getDocumentType("child"); + assertTrue(dt != null); + dt = docMan.getDocumentType("parent"); + assertTrue(dt != null); + dt = docMan.getDocumentType("grandparent"); + assertTrue(dt != null); + } + + @Test + public void can_convert_structs_and_annotations() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/converter/child.sd"); + collection.addSchemaFromFile("src/test/converter/other.sd"); + collection.addSchemaFromFile("src/test/converter/parent.sd"); + collection.addSchemaFromFile("src/test/converter/grandparent.sd"); + collection.resolveInternalConnections(); + var docMan = new DocumentTypeManager(); + var converter = new ConvertSchemaCollection(collection, docMan); + converter.convertTypes(); + var dt = docMan.getDocumentType("child"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + dt = docMan.getDocumentType("parent"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + dt = docMan.getDocumentType("grandparent"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + dt = docMan.getDocumentType("other"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java index 1ee7ae4937a..da5d0da146f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java @@ -24,14 +24,14 @@ public class IntermediateCollectionTestCase { public void can_add_minimal_schema() throws Exception { String input = joinLines ("schema foo {", - " document bar {", + " document foo {", " }", "}"); var collection = new IntermediateCollection(); ParsedSchema schema = collection.addSchemaFromString(input); assertEquals("foo", schema.name()); assertTrue(schema.hasDocument()); - assertEquals("bar", schema.getDocument().name()); + assertEquals("foo", schema.getDocument().name()); } @Test @@ -153,9 +153,9 @@ public class IntermediateCollectionTestCase { @Test public void can_detect_schema_inheritance_cycles() throws Exception { var collection = new IntermediateCollection(); - collection.addSchemaFromString("schema foo inherits bar {}"); - collection.addSchemaFromString("schema bar inherits qux {}"); - collection.addSchemaFromString("schema qux inherits foo {}"); + collection.addSchemaFromString("schema foo inherits bar { document foo {} }"); + collection.addSchemaFromString("schema bar inherits qux { document bar {} }"); + collection.addSchemaFromString("schema qux inherits foo { document qux {} }"); assertEquals(collection.getParsedSchemas().size(), 3); var ex = assertThrows(IllegalArgumentException.class, () -> collection.resolveInternalConnections()); @@ -171,6 +171,7 @@ public class IntermediateCollectionTestCase { assertEquals(collection.getParsedSchemas().size(), 3); var ex = assertThrows(IllegalArgumentException.class, () -> collection.resolveInternalConnections()); + System.err.println("ex: "+ex.getMessage()); assertTrue(ex.getMessage().startsWith("Inheritance cycle for documents: ")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java index 4379976ce64..6f04b37bb5b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java @@ -35,13 +35,13 @@ public class IntermediateParserTestCase { public void minimal_schema_can_be_parsed() throws Exception { String input = joinLines ("schema foo {", - " document bar {", + " document foo {", " }", "}"); ParsedSchema schema = parseString(input); assertEquals("foo", schema.name()); assertTrue(schema.hasDocument()); - assertEquals("bar", schema.getDocument().name()); + assertEquals("foo", schema.getDocument().name()); } @Test @@ -59,13 +59,13 @@ public class IntermediateParserTestCase { public void multiple_documents_disallowed() throws Exception { String input = joinLines ("schema foo {", - " document foo1 {", + " document foo {", " }", " document foo2 {", " }", "}"); var e = assertThrows(IllegalArgumentException.class, () -> parseString(input)); - assertEquals("schema 'foo' error: already has document foo1 so cannot add document foo2", e.getMessage()); + assertEquals("schema 'foo' error: already has document foo so cannot add document foo2", e.getMessage()); } void checkFileParses(String fileName) throws Exception { |