diff options
Diffstat (limited to 'config-model/src')
39 files changed, 187 insertions, 485 deletions
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index d98869e9dd3..4ff54d7ff1c 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -4,8 +4,10 @@ package com.yahoo.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.Field; +import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.datatypes.FieldValue; @@ -383,4 +385,18 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } + private ReferenceDataType refToThis = null; + + @SuppressWarnings("deprecation") + public ReferenceDataType getReferenceDataType() { + if (refToThis == null) { + // super ugly, the APIs for this are horribly inconsistent + var tmptmp = TemporaryStructuredDataType.create(getName()); + var tmp = ReferenceDataType.createWithInferredId(tmptmp); + tmp.setTargetType((StructuredDataType) this); + refToThis = tmp; + } + return refToThis; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index 16eef798acd..64688a7e70d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -84,7 +84,6 @@ public class Application { List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas); for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { - new FieldOperationApplierForStructs().processSchemaFields(schema); new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? new Processing(properties).process(schema, logger, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 4a449dc898f..2d9c81085fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -292,17 +292,8 @@ public class DocumentModelBuilder { else if (type instanceof ReferenceDataType) { ReferenceDataType t = (ReferenceDataType) type; var tt = t.getTargetType(); - if (tt instanceof TemporaryStructuredDataType) { - DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements); - t.setTargetType((StructuredDataType) targetType); - } else if (tt instanceof DocumentType) { - DataType targetType = resolveTemporariesRecurse(tt, repo, docs, replacements); - // super ugly, the APIs for this are horribly inconsistent - var tmptmp = TemporaryStructuredDataType.create(tt.getName()); - var tmp = new ReferenceDataType(tmptmp, t.getId()); - tmp.setTargetType((StructuredDataType) targetType); - type = tmp; - } + var doc = getDocumentType(docs, tt.getId()); + type = doc.getReferenceDataType(); } if (type != original) { replacements.add(new TypeReplacement(original, type)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 5e5623e2319..4a5a858f828 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -20,47 +20,8 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { for (SDDocumentType type : sdoc.getAllTypes()) { if (type.isStruct()) { apply(type); - copyFields(type, sdoc); } } } - @SuppressWarnings("deprecation") - private void copyFields(SDDocumentType structType, SDDocumentType sdoc) { - //find all fields in OTHER types that have this type: - List<SDDocumentType> list = new ArrayList<>(); - list.add(sdoc); - list.addAll(sdoc.getTypes()); - for (SDDocumentType anyType : list) { - Iterator<Field> fields = anyType.fieldIterator(); - while (fields.hasNext()) { - SDField field = (SDField) fields.next(); - maybePopulateField(sdoc, field, structType); - } - } - } - - private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) { - DataType structUsedByField = field.getFirstStructRecursive(); - if (structUsedByField == null) { - return; - } - if (structUsedByField.getName().equals(structType.getName())) { - //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); - field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); - } - } - - public void processSchemaFields(Schema schema) { - var sdoc = schema.getDocument(); - if (sdoc == null) return; - for (SDDocumentType type : sdoc.getAllTypes()) { - if (type.isStruct()) { - for (SDField field : schema.allExtraFields()) { - maybePopulateField(sdoc, field, type); - } - } - } - } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index a037d5046a7..b87bdd8907e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -268,8 +268,8 @@ public class SDDocumentType implements Cloneable, Serializable { return field; } - public Field addField(String string, DataType dataType, boolean header, int code) { - SDField field = new SDField(this, string, code, dataType, header); + public Field addField(String fName, DataType dataType, boolean header, int code) { + SDField field = new SDField(this, fName, code, dataType); addField(field); return field; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index bd6625d4bde..8263352e87f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -111,8 +111,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** Struct fields defined in this field */ private final Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0); - /** The document that this field was declared in, or null*/ - private SDDocumentType ownerDocType = null; + /** The document that this field was declared in, or null */ + private SDDocumentType repoDocType = null; /** The aliases declared for this field. May pertain to indexes or attributes */ private final Map<String, String> aliasToName = new HashMap<>(); @@ -130,25 +130,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * @param name the name of the field * @param dataType the datatype of the field */ - protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { super(name, id, dataType); - populate(populate, repo, name, dataType); + this.repoDocType = repo; + populate(name, dataType); } - public SDField(SDDocumentType repo, String name, int id, DataType dataType) { - this(repo, name, id, dataType, true); + public SDField(String name, DataType dataType) { + this(null, name, dataType); } /** Creates a new field */ - public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { - super(name, dataType); - populate(populate, repo, name, dataType); + public SDField(SDDocumentType repo, String name, DataType dataType) { + this(repo, name, dataType, null); } /** Creates a new field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { - super(name, dataType, owner == null ? null : owner.getDocumentType()); - populate(populate, repo, name, dataType); + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + this(repo, name, dataType, owner, null, 0); } /** @@ -159,27 +158,24 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * @param owner the owning document (used to check for id collisions) * @param fieldMatching the matching object to set for the field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, - Matching fieldMatching, boolean populate, int recursion) { + protected SDField(SDDocumentType repo, + String name, + DataType dataType, + SDDocumentType owner, + Matching fieldMatching, + int recursion) + { super(name, dataType, owner == null ? null : owner.getDocumentType()); + this.repoDocType = repo; + this.structFieldDepth = recursion; if (fieldMatching != null) this.setMatching(fieldMatching); - populate(populate, repo, name, dataType, fieldMatching, recursion); + populate(name, dataType); } - public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name, dataType, true); - } + private int structFieldDepth = 0; - public SDField(String name, DataType dataType) { - this(null, name, dataType); - } - - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { - populate(populate, repo, name, dataType, null, 0); - } - - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { + private void populate(String name, DataType dataType) { if (dataType instanceof TensorDataType) { TensorType type = ((TensorDataType)dataType).getTensorType(); if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) @@ -194,10 +190,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, else { addQueryCommand("type " + dataType.getName()); } - if (populate || (dataType instanceof MapDataType)) { - populateWithStructFields(repo, name, dataType, recursion); - populateWithStructMatching(repo, dataType, fieldMatching); - } } public void setIsExtraField(boolean isExtra) { @@ -273,17 +265,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } + private boolean doneStructFields = false; + @SuppressWarnings("deprecation") - public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { - DataType dt = getFirstStructOrMapRecursive(); - if (dt == null) return; + private void actuallyMakeStructFields() { + if (doneStructFields) return; + if (getFirstStructOrMapRecursive() == null) { + doneStructFields = true; + return; + } + var sdoc = repoDocType; + var dataType = getDataType(); java.util.function.BiConsumer<String, DataType> supplyStructField = (fieldName, fieldType) -> { if (structFields.containsKey(fieldName)) return; - String subName = name.concat(".").concat(fieldName); - var subField = new SDField(sdoc, subName, fieldType, - ownerDocType, new Matching(), - true, recursion + 1); + String subName = getName().concat(".").concat(fieldName); + var subField = new SDField(sdoc, subName, fieldType, null, + null, structFieldDepth + 1); structFields.put(fieldName, subField); }; @@ -292,15 +290,16 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, supplyStructField.accept("key", mdt.getKeyType()); supplyStructField.accept("value", mdt.getValueType()); } else { - if (recursion >= 10) return; + if (structFieldDepth >= 10) { + // too risky, infinite recursion + doneStructFields = true; + return; + } if (dataType instanceof CollectionDataType) { dataType = ((CollectionDataType)dataType).getNestedType(); } - if (dataType instanceof TemporaryStructuredDataType) { - SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; - if (subType == null) { - throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'."); - } + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (dataType instanceof TemporaryStructuredDataType && subType != null) { for (Field field : subType.fieldSet()) { supplyStructField.accept(field.getName(), field.getDataType()); } @@ -310,37 +309,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, supplyStructField.accept(field.getName(), field.getDataType()); } } - } - } - - public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) { - if (sdoc == null) return; - if (superFieldMatching == null) return; - DataType dt = getFirstStructOrMapRecursive(); - if (dt == null) return; - - if (dataType instanceof MapDataType) { - // old code here would never do anything useful, should we do something here? - return; - } else { - if (dataType instanceof CollectionDataType) { - dataType = ((CollectionDataType)dataType).getNestedType(); + if ((subType == null) && (structFields.size() > 0)) { + throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + + this + " [" + getDataType() + getDataType().getClass() + + "] with " + structFields.size() + " struct fields"); } - if (dataType instanceof StructDataType) { - SDDocumentType subType = sdoc.getType(dataType.getName()); - if (subType == null) { - throw new IllegalArgumentException("Could not find struct " + dataType.getName()); - } + // populate struct fields with matching + if (subType != null) { for (Field f : subType.fieldSet()) { if (f instanceof SDField) { SDField field = (SDField) f; Matching subFieldMatching = new Matching(); - subFieldMatching.merge(superFieldMatching); + subFieldMatching.merge(this.matching); subFieldMatching.merge(field.getMatching()); SDField subField = structFields.get(field.getName()); if (subField != null) { - subFieldMatching.merge(subField.getMatching()); - subField.populateWithStructMatching(sdoc, field.getDataType(), subFieldMatching); + // we just made this with no matching, so nop: + // subFieldMatching.merge(subField.getMatching()); subField.setMatching(subFieldMatching); } } else { @@ -349,8 +334,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } } + doneStructFields = true; } + private Matching matchingForStructFields = null; + public void addOperation(FieldOperation op) { pendingOperations.add(op); } @@ -723,7 +711,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** Returns list of static struct fields */ @Override - public Collection<SDField> getStructFields() { return structFields.values(); } + public Collection<SDField> getStructFields() { + actuallyMakeStructFields(); + return structFields.values(); + } /** * Returns a struct field defined in this field, @@ -732,6 +723,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, */ @Override public SDField getStructField(String name) { + actuallyMakeStructFields(); if (name.contains(".")) { String superFieldName = name.substring(0,name.indexOf(".")); String subFieldName = name.substring(name.indexOf(".")+1); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 4ced104fa55..8c17b607f94 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -8,12 +8,12 @@ import com.yahoo.document.DataType; */ public class TemporarySDField extends SDField { - public TemporarySDField(String name, DataType dataType, SDDocumentType owner) { - super(owner, name, dataType, owner, false); + public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + super(repo, name, dataType, owner); } - public TemporarySDField(String name, DataType dataType) { - super(null, name, dataType, false); + public TemporarySDField(SDDocumentType repo, String name, DataType dataType) { + super(repo, name, dataType); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java index fe3ac11af27..a5f5f961ab5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java @@ -24,6 +24,8 @@ public class IndexingOperation implements FieldOperation { this.script = script; } + public ScriptExpression getScript() { return script; } + public void apply(SDField field) { field.setIndexingScript(script); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java index 36c11b33b23..caeebd65f4f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java @@ -291,13 +291,12 @@ public class ConvertParsedFields { schema.addIndex(index); } - SDDocumentType convertStructDeclaration(Schema schema, ParsedStruct parsed) { + SDDocumentType convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) { // TODO - can we cleanup this mess var structProxy = new SDDocumentType(parsed.name(), schema); - structProxy.setStruct(context.resolveStruct(parsed)); for (var parsedField : parsed.getFields()) { var fieldType = context.resolveType(parsedField.getType()); - var field = new SDField(structProxy, parsedField.name(), fieldType); + var field = new SDField(document, parsedField.name(), fieldType); convertCommonFieldSettings(field, parsedField); structProxy.addField(field); if (parsedField.hasIdOverride()) { @@ -307,6 +306,7 @@ public class ConvertParsedFields { for (String inherit : parsed.getInherited()) { structProxy.inherit(new DataTypeName(inherit)); } + structProxy.setStruct(context.resolveStruct(parsed)); return structProxy; } @@ -314,7 +314,7 @@ public class ConvertParsedFields { var annType = context.resolveAnnotation(parsed.name()); var payload = parsed.getStruct(); if (payload.isPresent()) { - var structProxy = convertStructDeclaration(schema, payload.get()); + var structProxy = convertStructDeclaration(schema, document, payload.get()); document.addType(structProxy); } document.addAnnotation(annType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index 67e6c88d043..2d9a788cfef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -159,7 +159,7 @@ public class ConvertSchemaCollection { document.inherit(parent); } for (var struct : parsed.getStructs()) { - var structProxy = fieldConverter.convertStructDeclaration(schema, struct); + var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct); document.addType(structProxy); } for (var annotation : parsed.getAnnotations()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index 51defffa00b..0be48d1fd25 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -70,7 +70,7 @@ public class AddExtraFieldsToDocument extends Processor { if (docField == null) { ImmutableSDField existingField = schema.getField(field.getName()); if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType(), true); + SDField newField = new SDField(document, field.getName(), field.getDataType()); newField.setIsExtraField(true); document.addField(newField); } else if (!existingField.isImportedField()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index 0bb1b7da769..d7882c7f8fb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -10,6 +10,7 @@ import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.GeoPos; +import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -35,8 +36,11 @@ import java.util.logging.Level; */ public class CreatePositionZCurve extends Processor { + private final SDDocumentType repo; + public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); + this.repo = schema.getDocument(); } private boolean useV8GeoPositions = false; @@ -105,7 +109,7 @@ public class CreatePositionZCurve extends Processor { "' already created."); } boolean isArray = inputField.getDataType() instanceof ArrayDataType; - SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); + SDField field = new SDField(repo, fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY : Attribute.CollectionType.SINGLE); attribute.setPosition(true); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index 84dc6d369fc..7397f9a289c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -61,7 +61,7 @@ public class UriHack extends Processor { String partName = uriName + "." + suffix; // I wonder if this is explicit in qrs or implicit in backend? // search.addFieldSetItem(uriName, partName); - SDField partField = new SDField(partName, generatedType); + SDField partField = new SDField(schema.getDocument(), partName, generatedType); partField.setIndexStructureField(uriField.doesIndexing()); partField.setRankType(uriField.getRankType()); partField.setStemming(Stemming.NONE); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 8390cc59b6f..ac99bee93ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -108,6 +108,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri public static final Logger log = Logger.getLogger(VespaModel.class.getName()); private final Version version; + private final Version wantedNodeVersion; private final ConfigModelRepo configModelRepo = new ConfigModelRepo(); private final AllocatedHosts allocatedHosts; @@ -170,6 +171,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri throws IOException, SAXException { super("vespamodel"); version = deployState.getVespaVersion(); + wantedNodeVersion = deployState.getWantedNodeVespaVersion(); fileReferencesRepository = new FileReferencesRepository(deployState.getFileRegistry()); rankingConstants = new RankingConstants(deployState.getFileRegistry(), Optional.empty()); validationOverrides = deployState.validationOverrides(); @@ -407,6 +409,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return version; } + @Override + public Version wantedNodeVersion() { + return wantedNodeVersion; + } + /** * Resolves config of the given type and config id, by first instantiating the correct {@link com.yahoo.config.ConfigInstance.Builder}, * calling {@link #getConfig(com.yahoo.config.ConfigInstance.Builder, String)}. The default values used will be those of the config diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java index 2742dc59fcd..88139de7888 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java @@ -36,15 +36,13 @@ import java.util.stream.Collectors; */ public class OnnxModelInfo { - private final ApplicationPackage app; private final String modelPath; private final String defaultOutput; private final Map<String, OnnxTypeInfo> inputs; private final Map<String, OnnxTypeInfo> outputs; private final Map<String, TensorType> vespaTypes = new HashMap<>(); - private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs, Map<String, OnnxTypeInfo> outputs, String defaultOutput) { - this.app = app; + private OnnxModelInfo(String path, Map<String, OnnxTypeInfo> inputs, Map<String, OnnxTypeInfo> outputs, String defaultOutput) { this.modelPath = path; this.inputs = Collections.unmodifiableMap(inputs); this.outputs = Collections.unmodifiableMap(outputs); @@ -81,15 +79,7 @@ public class OnnxModelInfo { Set<Long> unboundSizes = new HashSet<>(); Map<String, Long> symbolicSizes = new HashMap<>(); resolveUnknownDimensionSizes(inputTypes, symbolicSizes, unboundSizes); - - TensorType type = TensorType.empty; - if (inputTypes.size() > 0 && onnxTypeInfo.needModelProbe(symbolicSizes)) { - type = OnnxModelProbe.probeModel(app, Path.fromString(modelPath), onnxName, inputTypes); - } - if (type.equals(TensorType.empty)) { - type = onnxTypeInfo.toVespaTensorType(symbolicSizes, unboundSizes); - } - return type; + return onnxTypeInfo.toVespaTensorType(symbolicSizes, unboundSizes); } return vespaTypes.computeIfAbsent(onnxName, v -> onnxTypeInfo.toVespaTensorType()); } @@ -160,8 +150,7 @@ public class OnnxModelInfo { Onnx.ModelProto model = Onnx.ModelProto.parseFrom(inputStream); String json = onnxModelToJson(model, path); storeGeneratedInfo(json, path, app); - return jsonToModelInfo(json, app); - + return jsonToModelInfo(json); } catch (IOException e) { throw new IllegalArgumentException("Unable to parse ONNX model", e); } @@ -170,7 +159,7 @@ public class OnnxModelInfo { static private OnnxModelInfo loadFromGeneratedInfo(Path path, ApplicationPackage app) { try { String json = readGeneratedInfo(path, app); - return jsonToModelInfo(json, app); + return jsonToModelInfo(json); } catch (IOException e) { throw new IllegalArgumentException("Unable to parse ONNX model", e); } @@ -213,7 +202,7 @@ public class OnnxModelInfo { return out.toString(); } - static public OnnxModelInfo jsonToModelInfo(String json, ApplicationPackage app) throws IOException { + static public OnnxModelInfo jsonToModelInfo(String json) throws IOException { ObjectMapper m = new ObjectMapper(); JsonNode root = m.readTree(json); Map<String, OnnxTypeInfo> inputs = new HashMap<>(); @@ -233,7 +222,7 @@ public class OnnxModelInfo { if (root.get("outputs").has(0)) { defaultOutput = root.get("outputs").get(0).get("name").textValue(); } - return new OnnxModelInfo(app, path, inputs, outputs, defaultOutput); + return new OnnxModelInfo(path, inputs, outputs, defaultOutput); } static private void onnxTypeToJson(JsonGenerator g, Onnx.ValueInfoProto valueInfo) throws IOException { @@ -364,21 +353,6 @@ public class OnnxModelInfo { return builder.build(); } - boolean needModelProbe(Map<String, Long> symbolicSizes) { - for (OnnxDimensionInfo onnxDimension : dimensions) { - if (onnxDimension.hasSymbolicName()) { - if (symbolicSizes == null) - return true; - if ( ! symbolicSizes.containsKey(onnxDimension.getSymbolicName())) { - return true; - } - } else if (onnxDimension.getSize() == 0) { - return true; - } - } - return false; - } - @Override public String toString() { return "(" + valueType.id() + ")" + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java deleted file mode 100644 index 2e2ebdeb98f..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ /dev/null @@ -1,152 +0,0 @@ -package com.yahoo.vespa.model.ml; - -import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.tensor.TensorType; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -/** - * Defers to 'vespa-analyze-onnx-model' to determine the output type given - * a set of inputs. For situations with symbolic dimension sizes that can't - * easily be determined. - * - * @author lesters - */ -public class OnnxModelProbe { - - private static final String binary = "vespa-analyze-onnx-model"; - - static TensorType probeModel(ApplicationPackage app, Path modelPath, String outputName, Map<String, TensorType> inputTypes) { - TensorType outputType = TensorType.empty; - String contextKey = createContextKey(outputName, inputTypes); - - try { - // Check if output type has already been probed - outputType = readProbedOutputType(app, modelPath, contextKey); - - // Otherwise, run vespa-analyze-onnx-model if the model is available - if (outputType.equals(TensorType.empty) && app.getFile(modelPath).exists()) { - String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); - String jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); - outputType = outputTypeFromJson(jsonOutput, outputName); - if ( ! outputType.equals(TensorType.empty)) { - writeProbedOutputType(app, modelPath, contextKey, outputType); - } - } - - } catch (IllegalArgumentException | IOException | InterruptedException e) { - e.printStackTrace(System.err); - } - - return outputType; - } - - private static String createContextKey(String onnxName, Map<String, TensorType> inputTypes) { - StringBuilder key = new StringBuilder().append(onnxName).append(":"); - inputTypes.entrySet().stream().sorted(Map.Entry.comparingByKey()) - .forEachOrdered(e -> key.append(e.getKey()).append(":").append(e.getValue()).append(",")); - return key.substring(0, key.length()-1); - } - - private static Path probedOutputTypesPath(Path path) { - String fileName = OnnxModelInfo.asValidIdentifier(path.getRelative()) + ".probed_output_types"; - return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); - } - - static void writeProbedOutputType(ApplicationPackage app, Path modelPath, String output, - Map<String, TensorType> inputTypes, TensorType type) throws IOException { - writeProbedOutputType(app, modelPath, createContextKey(output, inputTypes), type); - } - - private static void writeProbedOutputType(ApplicationPackage app, Path modelPath, - String contextKey, TensorType type) throws IOException { - String path = app.getFileReference(probedOutputTypesPath(modelPath)).getAbsolutePath(); - IOUtils.writeFile(path, contextKey + "\t" + type + "\n", true); - } - - private static TensorType readProbedOutputType(ApplicationPackage app, Path modelPath, - String contextKey) throws IOException { - ApplicationFile file = app.getFile(probedOutputTypesPath(modelPath)); - if ( ! file.exists()) { - return TensorType.empty; - } - try (BufferedReader reader = new BufferedReader(file.createReader())) { - String line; - while (null != (line = reader.readLine())) { - String[] parts = line.split("\t"); - String key = parts[0]; - if (key.equals(contextKey)) { - return TensorType.fromSpec(parts[1]); - } - } - } - return TensorType.empty; - } - - private static TensorType outputTypeFromJson(String json, String outputName) throws IOException { - ObjectMapper m = new ObjectMapper(); - JsonNode root = m.readTree(json); - if ( ! root.isObject() || ! root.has("outputs")) { - return TensorType.empty; - } - JsonNode outputs = root.get("outputs"); - if ( ! outputs.has(outputName)) { - return TensorType.empty; - } - return TensorType.fromSpec(outputs.get(outputName).asText()); - } - - private static String createJsonInput(String modelPath, Map<String, TensorType> inputTypes) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); - g.writeStartObject(); - g.writeStringField("model", modelPath); - g.writeObjectFieldStart("inputs"); - for (Map.Entry<String, TensorType> input : inputTypes.entrySet()) { - g.writeStringField(input.getKey(), input.getValue().toString()); - } - g.writeEndObject(); - g.writeEndObject(); - g.close(); - return out.toString(); - } - - private static String callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { - ProcessBuilder processBuilder = new ProcessBuilder(binary, "--probe-types"); - StringBuilder output = new StringBuilder(); - Process process = processBuilder.start(); - - // Write json array to process stdin - OutputStream os = process.getOutputStream(); - os.write(jsonInput.getBytes(StandardCharsets.UTF_8)); - os.close(); - - // Read output from stdout - InputStream inputStream = process.getInputStream(); - while (true) { - int b = inputStream.read(); - if (b == -1) break; - output.append((char)b); - } - int returnCode = process.waitFor(); - if (returnCode != 0) { - throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". Output:\n" + output); - } - return output.toString(); - } - -} diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 7447eee5cec..ba955f071b2 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -958,10 +958,7 @@ void indexingOperation(ParsedField field, boolean multiLine) : { } { { IndexingOperation oldOp = newIndexingOperation(multiLine); - // TODO conversion via SDField is very ugly - SDField tmpField = new SDField("temp", com.yahoo.document.DataType.STRING); - oldOp.apply(tmpField); - ParsedIndexingOp newOp = new ParsedIndexingOp(tmpField.getIndexingScript()); + ParsedIndexingOp newOp = new ParsedIndexingOp(oldOp.getScript()); field.setIndexingOperation(newOp); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 1a0b518bbb2..e90df2776e0 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -639,7 +639,7 @@ void field(SDDocumentType document, Schema schema) : if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, document); + field = new TemporarySDField(document == null ? schema.getDocument() : document, name, type, document); } lbrace() (fieldBody(field, schema, document) (<NL>)*)* <RBRACE> { @@ -668,7 +668,7 @@ void fieldSet(Schema schema) : | ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); } | - ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } + ( matchSetting = match(new SDField(null, setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } )(<NL>)*)+ <RBRACE> { @@ -698,11 +698,13 @@ void annotationOutside(Schema schema) : <ANNOTATION> name = identifier() { type = new SDAnnotationType(name.trim()); + if (schema.getDocument() == null) { + throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one."); + } } [ inheritsAnnotation(type) (<NL>)* ] - lbrace() (type = annotationBody(schema, type)) <RBRACE> + lbrace() (type = annotationBody(schema, schema.getDocument(), type)) <RBRACE> { - if (schema.getDocument()==null) throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one."); schema.addAnnotation(type); } } @@ -723,7 +725,7 @@ void annotation(Schema schema, SDDocumentType document) : type = new SDAnnotationType(name.trim()); } [ inheritsAnnotation(type) (<NL>)* ] - lbrace() (type = annotationBody(schema, type)) <RBRACE> + lbrace() (type = annotationBody(schema, document, type)) <RBRACE> { document.addAnnotation(type); } @@ -737,12 +739,12 @@ void annotation(Schema schema, SDDocumentType document) : * @param type the type being built * @return a modified or new AnnotationType instance */ -SDAnnotationType annotationBody(Schema schema, SDAnnotationType type) : +SDAnnotationType annotationBody(Schema schema, SDDocumentType repo, SDAnnotationType type) : { SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema); } { - (structFieldDefinition(struct) (<NL>)*)* + (structFieldDefinition(repo, struct) (<NL>)*)* { if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField. type = new SDAnnotationType(type.getName(), struct, type.getInherits()); @@ -809,9 +811,13 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : SDDocumentType struct; } { - ( <STRUCT> name = identifier() (<NL>)* { struct = new SDDocumentType(name, schema); } + ( <STRUCT> name = identifier() (<NL>)* { + if (repo == null) + throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one."); + struct = new SDDocumentType(name, schema); + } [ inheritsDocument(struct) (<NL>)* ] - lbrace() (structFieldDefinition(struct) (<NL>)*)* <RBRACE> ) + lbrace() (structFieldDefinition(repo, struct) (<NL>)*)* <RBRACE> ) { try { docMan.getDataType(name); @@ -819,7 +825,6 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : } catch (IllegalArgumentException e) { // empty } - if (repo==null) throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one."); SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName()); DataType stype = sdtype != null ? sdtype.getStruct() @@ -828,7 +833,7 @@ SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : return struct; } } - + /** * This rule consumes a data type block from within a field element. * @@ -921,7 +926,7 @@ DataType wildCardType() : * * @param struct The struct to modify. */ -void structFieldDefinition(SDDocumentType struct) : +void structFieldDefinition(SDDocumentType document, SDDocumentType struct) : { String name; SDField field; @@ -932,7 +937,7 @@ void structFieldDefinition(SDDocumentType struct) : if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, struct); + field = new TemporarySDField(document, name, type, struct); struct.addField(field); } lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> { diff --git a/config-model/src/test/cfg/application/onnx_probe/files/create_dynamic_model.py b/config-model/src/test/cfg/application/onnx_probe/files/create_dynamic_model.py deleted file mode 100755 index b493e394ee4..00000000000 --- a/config-model/src/test/cfg/application/onnx_probe/files/create_dynamic_model.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import onnx -import numpy as np -from onnx import helper, TensorProto - -INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT, ["batch", 2]) -INPUT_2 = helper.make_tensor_value_info('input2', TensorProto.FLOAT, ["batch", 2]) -OUTPUT = helper.make_tensor_value_info('out', TensorProto.FLOAT, ["batch", "dim1", "dim2"]) - -SHAPE = helper.make_tensor('shape', TensorProto.INT64, dims=[3], vals=np.array([1,2,2]).astype(np.int64)) - -nodes = [ - helper.make_node('Concat', ['input1', 'input2'], ['concat'], axis=1), - helper.make_node('Reshape', ['concat', 'shape'], ['out']), -] -graph_def = helper.make_graph(nodes, 'simple_scoring', [INPUT_1, INPUT_2], [OUTPUT], [SHAPE]) -model_def = helper.make_model(graph_def, producer_name='create_dynamic_model.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) -onnx.save(model_def, 'dynamic_model_2.onnx') diff --git a/config-model/src/test/cfg/application/onnx_probe/files/dynamic_model.onnx b/config-model/src/test/cfg/application/onnx_probe/files/dynamic_model.onnx deleted file mode 100644 index 28c600e2a09..00000000000 --- a/config-model/src/test/cfg/application/onnx_probe/files/dynamic_model.onnx +++ /dev/null @@ -1,21 +0,0 @@ -create_dynamic_model_2.py:Ö -- -input1 -input2concat"Concat* -axis - -concat -shapeout"Reshapesimple_scoring*:BshapeZ -input1 -
-batch -Z -input2 -
-batch -b& -out - -batch -dim1 -dim2B
\ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index 876ed00d0c4..ef1bb4c5ad4 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -47,8 +47,6 @@ doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" doctype[1].documentref[0].idx 10017 doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10018 doctype[1].structtype[0].idx 10016 doctype[1].structtype[0].name "ad.header" doctype[1].structtype[0].field[0].name "campaign_ref" @@ -56,10 +54,10 @@ doctype[1].structtype[0].field[0].internalid 23963250 doctype[1].structtype[0].field[0].type 10017 doctype[1].structtype[0].field[1].name "other_campaign_ref" doctype[1].structtype[0].field[1].internalid 874751172 -doctype[1].structtype[0].field[1].type 10019 +doctype[1].structtype[0].field[1].type 10017 doctype[2].name "campaign" doctype[2].idx 10018 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10020 -doctype[2].structtype[0].idx 10020 +doctype[2].contentstruct 10019 +doctype[2].structtype[0].idx 10019 doctype[2].structtype[0].name "campaign.header" diff --git a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd index 32823e46592..0636e7a537e 100644 --- a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd +++ b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd @@ -21,6 +21,7 @@ search streamingstruct { # Allow default matchtypes in struct. Can be overridden. # No index/attribute related stuff. It is only a datatype definition. } + struct ns1 { field nf1 type s1 { } field nf1s type s1 { match: substring } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java index 81a44261daf..8edbd789a27 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java @@ -156,8 +156,8 @@ public class DocumentGraphValidatorTest { @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); fromDocument.addField(refField); Map<String, DocumentReference> originalMap = fromDocument.getDocumentReferences().get().referenceMap(); HashMap<String, DocumentReference> modifiedMap = new HashMap<>(originalMap); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 66f1850bd10..4fa145afae9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -38,12 +38,12 @@ public class DocumentReferenceResolverTest { barSchema.addDocument(barDocument); // Create foo document with document reference to bar and add another field - SDField fooRefToBarField = new SDField - ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(fooRefToBarField); - SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToBarField); + SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT); fooDocument.addField(fooRefToBarField); fooDocument.addField(irrelevantField); fooSchema.addDocument(fooDocument); @@ -62,11 +62,12 @@ public class DocumentReferenceResolverTest { @Test public void throws_user_friendly_exception_if_referenced_document_does_not_exist() { // Create foo document with document reference to non-existing document bar + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); SDField fooRefToBarField = new SDField( + fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); AttributeUtils.addAttributeAspect(fooRefToBarField); - Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); - SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); @@ -86,10 +87,10 @@ public class DocumentReferenceResolverTest { barSchema.addDocument(barDocument); // Create foo document with document reference to bar - SDField fooRefToBarField = new SDField - ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java index 7d2386030da..7e708f93a96 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -22,16 +22,18 @@ public class ImportedFieldsEnumeratorTest { String PARENT = "parent"; Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); - var parentField = new SDField("their_field", DataType.INT); + var parentField = new SDField(parentDocument, "their_field", DataType.INT); AttributeUtils.addAttributeAspect(parentField); parentDocument.addField(parentField); parentSchema.addDocument(parentDocument); String FOO = "foo"; Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + /* SDField fooRefToParent = new SDField( "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(fooRefToParent); + */ var fooImports = fooSchema.temporaryImportedFields().get(); fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); @@ -40,9 +42,11 @@ public class ImportedFieldsEnumeratorTest { String BAR = "bar"; Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); + /* SDField barRefToParent = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); AttributeUtils.addAttributeAspect(barRefToParent); + */ var barImports = barSchema.temporaryImportedFields().get(); barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java index c7ec2d9e9d1..652a06a6025 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java @@ -37,22 +37,22 @@ public class SDDocumentTypeOrdererTestCase { g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); - SDField aFieldTypeB = new TemporarySDField("atypeb", DataType.STRING); + SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING); a.addField(aFieldTypeB); - SDField bFieldTypeC = new TemporarySDField("btypec", DataType.STRING); + SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING); b.addField(bFieldTypeC); - SDField cFieldTypeG = new TemporarySDField("ctypeg", DataType.STRING); + SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING); c.addField(cFieldTypeG); - SDField gFieldTypeF = new TemporarySDField("gtypef", DataType.STRING); + SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING); g.addField(gFieldTypeF); - SDField fFieldTypeC = new TemporarySDField("ftypec", DataType.STRING); + SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING); f.addField(fFieldTypeC); - SDField dFieldTypeE = new TemporarySDField("dtypee", DataType.STRING); + SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING); d.addField(dFieldTypeE); types.add(a); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 237e6c8c992..a3123550efa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -25,11 +25,11 @@ public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType doc = new SDDocumentType("test"); schema.addDocument(doc); - doc.addField(new SDField("a", DataType.STRING)); - SDField field = new SDField("b", DataType.STRING); + doc.addField(new SDField(doc, "a", DataType.STRING)); + SDField field = new SDField(doc, "b", DataType.STRING); field.setLiteralBoost(500); doc.addField(field); - doc.addField(new SDField("c", DataType.STRING)); + doc.addField(new SDField(doc, "c", DataType.STRING)); schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 440f067dd00..ba485b7b96b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -29,7 +29,7 @@ public class IdTestCase extends AbstractExportingTestCase { Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); - SDField uri = new SDField("URI", DataType.URI); + SDField uri = new SDField(document, "URI", DataType.URI); uri.parseIndexingScript("{ summary | index }"); document.addField(uri); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java index 63e3df2205f..275c94465ae 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java @@ -54,12 +54,8 @@ public class MatchSettingsResolvingTestCase extends AbstractExportingTestCase { @Test public void testMapWithStructSettings() throws IOException, ParseException { - /* - * does not work - * does not pick up settings from struct declaration assertCorrectDeriving("matchsettings_map_wss", new TestProperties().setExperimentalSdParsing(false)); - */ assertCorrectDeriving("matchsettings_map_wss", new TestProperties().setExperimentalSdParsing(true)); } @@ -74,11 +70,8 @@ public class MatchSettingsResolvingTestCase extends AbstractExportingTestCase { @Test public void testMapAfter() throws IOException, ParseException { - /* fails with: - java.lang.IllegalArgumentException: Could not find struct 'elem'. assertCorrectDeriving("matchsettings_map_after", new TestProperties().setExperimentalSdParsing(false)); - */ assertCorrectDeriving("matchsettings_map_after", new TestProperties().setExperimentalSdParsing(true)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java index 34d33a00d9e..2522f8f56e2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java @@ -91,8 +91,8 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDField refField = new TemporarySDField(refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(to.getName()))); fromDocument.addField(refField); Map<String, DocumentReference> originalMap = fromDocument.getDocumentReferences().get().referenceMap(); HashMap<String, DocumentReference> modifiedMap = new HashMap<>(originalMap); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index dbb32e61144..62a79e49146 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -30,7 +30,7 @@ public class TypeConversionTestCase extends AbstractSchemaTestCase { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); - SDField a = new SDField("a", DataType.STRING); + SDField a = new SDField(document, "a", DataType.STRING); a.parseIndexingScript("{ index }"); document.addField(a); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java index 5ab5a8057e8..9c974225605 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java @@ -25,8 +25,9 @@ public class VsmFieldsTestCase { @Test public void reference_type_field_is_unsearchable() { Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); - schema.addDocument(new SDDocumentType("test")); - SDField refField = new TemporarySDField("ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); + var sdoc = new SDDocumentType("test"); + schema.addDocument(sdoc); + SDField refField = new TemporarySDField(sdoc, "ref_field", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("parent_type"))); refField.parseIndexingScript("{ summary }"); schema.getDocument().addField(refField); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index 9ca97f4dbc7..0adaba4cf68 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -59,8 +59,9 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { } private static ImportedFields createSingleImportedField(String fieldName) { - Schema targetSchema = createSearch("target_doc"); - SDField targetField = new SDField("target_field", DataType.INT); + Schema targetSchema = createSearchWithDocument("target_doc"); + var doc = targetSchema.getDocument(); + SDField targetField = new SDField(doc, "target_field", DataType.INT); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); return new ImportedFields(Collections.singletonMap(fieldName, importedField)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java index 5d2590a420d..926e26451d7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java @@ -186,9 +186,10 @@ public class AdjustPositionSummaryFieldsTestCase { private void createPositionField(Schema schema, boolean setupPosAttr, boolean setupBadAttr) { String ilScript = setupPosAttr ? "{ summary | attribute }" : "{ summary }"; - schema.getDocument().addField(createField("pos", PositionDataType.INSTANCE, ilScript)); + var doc = schema.getDocument(); + doc.addField(createField(doc, "pos", PositionDataType.INSTANCE, ilScript)); if (setupBadAttr) { - schema.getDocument().addField(createField("pos_zcurve", DataType.LONG, "{ attribute }")); + doc.addField(createField(doc, "pos_zcurve", DataType.LONG, "{ attribute }")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 324010f9e83..d1d3f4489ce 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -110,13 +110,14 @@ public class ImportedFieldsResolverTestCase { public SearchModel() { super(); grandParentSchema = createSearch("grandparent"); - grandParentSchema.getDocument().addField(createField("ancient_field", DataType.INT, "{ attribute }")); - - parentSchema.getDocument().addField(createField("attribute_field", DataType.INT, "{ attribute }")); - parentSchema.getDocument().addField(createField("attribute_and_index", DataType.INT, "{ attribute | index }")); - parentSchema.getDocument().addField(new TemporarySDField("not_attribute", DataType.INT)); - parentSchema.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); - parentSchema.getDocument().addField(createField("predicate_field", DataType.PREDICATE, "{ attribute }")); + var grandParentDoc = grandParentSchema.getDocument(); + grandParentDoc.addField(createField(grandParentDoc, "ancient_field", DataType.INT, "{ attribute }")); + var parentDoc = parentSchema.getDocument(); + parentDoc.addField(createField(parentDoc, "attribute_field", DataType.INT, "{ attribute }")); + parentDoc.addField(createField(parentDoc, "attribute_and_index", DataType.INT, "{ attribute | index }")); + parentDoc.addField(new TemporarySDField(parentDoc, "not_attribute", DataType.INT)); + parentDoc.addField(createField(parentDoc, "tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); + parentDoc.addField(createField(parentDoc, "predicate_field", DataType.PREDICATE, "{ attribute }")); addRefField(parentSchema, grandParentSchema, "ref"); addImportedField(parentSchema, "ancient_field", "ref", "ancient_field"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java index b14c7287537..3b4612ee87a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java @@ -37,19 +37,19 @@ public class ParentChildSearchModel { return result; } - protected static TemporarySDField createField(String name, DataType dataType, String indexingScript) { - TemporarySDField result = new TemporarySDField(name, dataType); + protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { + TemporarySDField result = new TemporarySDField(repo, name, dataType); result.parseIndexingScript(indexingScript); return result; } @SuppressWarnings("deprecation") - protected static SDField createRefField(String parentType, String fieldName) { - return new TemporarySDField(fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType))); + protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) { + return new TemporarySDField(repo, fieldName, ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create(parentType))); } protected static void addRefField(Schema child, Schema parent, String fieldName) { - SDField refField = createRefField(parent.getName(), fieldName); + SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName); child.getDocument().addField(refField); child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(), new DocumentReference(refField, parent)))); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index 22fd4e45c4a..8c801d9deaf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -65,8 +65,8 @@ public class ValidateFieldTypesTest { } private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) { - Schema targetSchema = createSearch("target_doc"); - SDField targetField = new SDField("target_field", dataType); + Schema targetSchema = createSearchWithDocument("target_doc"); + SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); return new ImportedFields(Collections.singletonMap(fieldName, importedField)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/OnnxModelProbeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/OnnxModelProbeTest.java deleted file mode 100644 index b2d4953f8e2..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/OnnxModelProbeTest.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.ml; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.tensor.TensorType; -import org.junit.Test; - -import java.io.IOException; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -public class OnnxModelProbeTest { - - @Test - public void testProbedOutputTypes() throws IOException { - - Path appDir = Path.fromString("src/test/cfg/application/onnx_probe"); - Path storedAppDir = appDir.append("copy"); - try { - FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir.toFile()); - Path modelPath = Path.fromString("files/dynamic_model.onnx"); - String output = "out"; - Map<String, TensorType> inputTypes = Map.of( - "input1", TensorType.fromSpec("tensor<float>(d0[1],d1[2])"), - "input2", TensorType.fromSpec("tensor<float>(d0[1],d1[2])")); - TensorType expected = TensorType.fromSpec("tensor<float>(d0[1],d1[2],d2[2])"); - - TensorType outputType = OnnxModelProbe.probeModel(app, modelPath, output, inputTypes); - - // if 'vespa-analyze-onnx-model' was unavailable, specifically cache expected type - if (outputType.equals(TensorType.empty)) { - OnnxModelProbe.writeProbedOutputType(app, modelPath, output, inputTypes, expected); - } else { - assertEquals(outputType, expected); - } - - // Test loading from generated info - storedAppDir.toFile().mkdirs(); - IOUtils.copyDirectory(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - app = FilesApplicationPackage.fromFile(storedAppDir.toFile()); - outputType = OnnxModelProbe.probeModel(app, modelPath, output, inputTypes); - assertEquals(outputType, expected); - - } finally { - IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - IOUtils.recursiveDeleteDir(storedAppDir.toFile()); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index e58c29cc4fd..405c1127842 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -42,7 +42,7 @@ public class SchemaClusterTest { // sd1 SDDocumentType sdt1 = new SDDocumentType("s1"); Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty()); - SDField f1 = new SDField("f1", DataType.STRING); + SDField f1 = new SDField(sdt1, "f1", DataType.STRING); f1.addAttribute(new Attribute("f1", DataType.STRING)); f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); sdt1.addField(f1); @@ -51,7 +51,7 @@ public class SchemaClusterTest { // sd2 SDDocumentType sdt2 = new SDDocumentType("s2"); Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty()); - SDField f2=new SDField("f2", DataType.STRING); + SDField f2=new SDField(sdt2, "f2", DataType.STRING); f2.addAttribute(new Attribute("f2", DataType.STRING)); f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); sdt2.addField(f2); |