diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/schema/document')
32 files changed, 3156 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java new file mode 100644 index 00000000000..f2279a52855 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -0,0 +1,435 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.Float16FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.tensor.TensorType; + +import java.io.Serializable; +import java.util.function.Supplier; +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * A search-time document attribute (per-document in-memory value). + * This belongs to the field defining the attribute. + * + * @author bratseth + */ +public final class Attribute implements Cloneable, Serializable { + + public enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT, HAMMING } + + // Remember to change hashCode and equals when you add new fields + + private String name; + + private Type type; + private CollectionType collectionType; + + private boolean removeIfZero = false; + private boolean createIfNonExistent = false; + private boolean enableBitVectors = false; + private boolean enableOnlyBitVector = false; + + private boolean fastRank = false; + private boolean fastSearch = false; + private boolean fastAccess = false; + private boolean huge = false; + private boolean mutable = false; + private boolean paged = false; + private int arity = BooleanIndexDefinition.DEFAULT_ARITY; + private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND; + private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND; + private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD; + + /** This is set if the type of this is TENSOR */ + private Optional<TensorType> tensorType = Optional.empty(); + + /** This is set if the type of this is REFERENCE */ + private final Optional<StructuredDataType> referenceDocumentType; + + private Optional<DistanceMetric> distanceMetric = Optional.empty(); + + private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); + + private boolean isPosition = false; + private final Sorting sorting = new Sorting(); + + /** The aliases for this attribute */ + private final Set<String> aliases = new LinkedHashSet<>(); + + private Dictionary dictionary = null; + private Case casing = Case.UNCASED; + + /** + * True if this attribute should be returned during first pass of search. + * Null means make the default decision for this kind of attribute + */ + private Boolean prefetch = null; + + /** The attribute type enumeration */ + public enum Type { + BYTE("byte", "INT8"), + SHORT("short", "INT16"), + INTEGER("integer", "INT32"), + LONG("long", "INT64"), + FLOAT16("float16", "FLOAT16"), + FLOAT("float", "FLOAT"), + DOUBLE("double", "DOUBLE"), + STRING("string", "STRING"), + BOOL("bool", "BOOL"), + PREDICATE("predicate", "PREDICATE"), + TENSOR("tensor", "TENSOR"), + REFERENCE("reference", "REFERENCE"); + + private final String myName; // different from what name() returns. + private final String exportAttributeTypeName; + + Type(String name, String exportAttributeTypeName) { + this.myName=name; + this.exportAttributeTypeName = exportAttributeTypeName; + } + + public String getName() { return myName; } + public String getExportAttributeTypeName() { return exportAttributeTypeName; } + + @Override + public String toString() { + return "type: " + myName; + } + } + + /** The attribute collection type enumeration */ + public enum CollectionType { + + SINGLE("SINGLE"), + ARRAY("ARRAY"), + WEIGHTEDSET ("WEIGHTEDSET"); + + private final String name; + + CollectionType(String name) { + this.name=name; + } + + public String getName() { return name; } + + @Override + public String toString() { + return "collectiontype: " + name; + } + + } + + /** Creates an attribute with default settings */ + public Attribute(String name, DataType fieldType) { + this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); + setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false); + setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false); + } + + public Attribute(String name, Type type, CollectionType collectionType) { + this(name, type, collectionType, Optional.empty(), Optional.empty()); + } + + public Attribute(String name, + Type type, + CollectionType collectionType, + Optional<TensorType> tensorType, + Optional<StructuredDataType> referenceDocumentType) { + this.name=name; + setType(type); + setCollectionType(collectionType); + this.tensorType = tensorType; + this.referenceDocumentType = referenceDocumentType; + } + + public Attribute convertToArray() { + Attribute result = clone(); + result.collectionType = CollectionType.ARRAY; + return result; + } + + /** + * <p>Returns whether this attribute should be included in the "attributeprefetch" summary + * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc. + * + * <p>Single value attributes are prefetched by default if summary is true. + * Multi value attributes are not.</p> + */ + public boolean isPrefetch() { + if (prefetch!=null) return prefetch.booleanValue(); + + if (CollectionType.SINGLE.equals(collectionType)) { + return true; + } + + return false; + } + + /** Returns the prefetch value of this, null if the default is used. */ + public Boolean getPrefetchValue() { return prefetch; } + + public boolean isRemoveIfZero() { return removeIfZero; } + public boolean isCreateIfNonExistent() { return createIfNonExistent; } + public boolean isEnabledBitVectors() { return enableBitVectors; } + public boolean isEnabledOnlyBitVector() { return enableOnlyBitVector; } + public boolean isFastSearch() { return fastSearch; } + public boolean isFastRank() { return fastRank; } + public boolean isFastAccess() { return fastAccess; } + public boolean isHuge() { return huge; } + public boolean isPaged() { return paged; } + public boolean isPosition() { return isPosition; } + public boolean isMutable() { return mutable; } + + public int arity() { return arity; } + public long lowerBound() { return lowerBound; } + public long upperBound() { return upperBound; } + public double densePostingListThreshold() { return densePostingListThreshold; } + public Optional<TensorType> tensorType() { return tensorType; } + public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } + + public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; + public DistanceMetric distanceMetric() { + return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); + } + public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } + + public Sorting getSorting() { return sorting; } + public Dictionary getDictionary() { return dictionary; } + public Case getCase() { return casing; } + + public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; } + public void setCreateIfNonExistent(boolean create) { this.createIfNonExistent = create; } + + /** + * Sets whether this should be included in the "attributeprefetch" document summary. + * True or false to override default, null to use default + */ + public void setPrefetch(Boolean prefetch) { this.prefetch = prefetch; } + public void setEnableBitVectors(boolean enableBitVectors) { this.enableBitVectors = enableBitVectors; } + public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; } + public void setFastRank(boolean value) { + Supplier<IllegalArgumentException> badGen = () -> new IllegalArgumentException("fast-rank is only valid for tensor attributes, invalid for: "+this); + var tt = tensorType.orElseThrow(badGen); + for (var dim : tt.dimensions()) { + if (dim.isMapped()) { + this.fastRank = value; + return; + } + } + throw badGen.get(); + } + public void setFastSearch(boolean fastSearch) { this.fastSearch = fastSearch; } + public void setHuge(boolean huge) { this.huge = huge; } + public void setPaged(boolean paged) { this.paged = paged; } + public void setFastAccess(boolean fastAccess) { this.fastAccess = fastAccess; } + public void setPosition(boolean position) { this.isPosition = position; } + public void setMutable(boolean mutable) { this.mutable = mutable; } + public void setArity(int arity) { this.arity = arity; } + public void setLowerBound(long lowerBound) { this.lowerBound = lowerBound; } + public void setUpperBound(long upperBound) { this.upperBound = upperBound; } + public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } + public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setDistanceMetric(DistanceMetric metric) { this.distanceMetric = Optional.of(metric); } + public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } + public void setDictionary(Dictionary dictionary) { this.dictionary = dictionary; } + public void setCase(Case casing) { this.casing = casing; } + + public String getName() { return name; } + public Type getType() { return type; } + public CollectionType getCollectionType() { return collectionType; } + + public void setName(String name) { this.name=name; } + private void setType(Type type) { this.type=type; } + public void setCollectionType(CollectionType type) { this.collectionType=type; } + + /** Converts to the right attribute type from a field datatype */ + public static Type convertDataType(DataType fieldType) { + if (fieldType instanceof NewDocumentReferenceDataType) { + return Type.REFERENCE; + } else if (fieldType instanceof CollectionDataType) { + return convertDataType(((CollectionDataType) fieldType).getNestedType()); + } + FieldValue fval = fieldType.createFieldValue(); + if (fval instanceof StringFieldValue) { + return Type.STRING; + } else if (fval instanceof IntegerFieldValue) { + return Type.INTEGER; + } else if (fval instanceof LongFieldValue) { + return Type.LONG; + } else if (fval instanceof FloatFieldValue) { + return Type.FLOAT; + } else if (fval instanceof DoubleFieldValue) { + return Type.DOUBLE; + } else if (fval instanceof BoolFieldValue) { + return Type.BOOL; + } else if (fval instanceof Float16FieldValue) { + return Type.FLOAT16; + } else if (fval instanceof ByteFieldValue) { + return Type.BYTE; + } else if (fval instanceof Raw) { + return Type.BYTE; + } else if (fval instanceof PredicateFieldValue) { + return Type.PREDICATE; + } else if (fval instanceof TensorFieldValue) { + return Type.TENSOR; + } else { + throw new IllegalArgumentException("Don't know which attribute type to convert " + + fieldType + " [" + fieldType.getClass() + "] to"); + } + } + + /** Converts to the right attribute type from a field datatype */ + private static CollectionType convertCollectionType(DataType fieldType) { + if (fieldType instanceof ArrayDataType) { + return CollectionType.ARRAY; + } else if (fieldType instanceof WeightedSetDataType) { + return CollectionType.WEIGHTEDSET; + } else if (fieldType instanceof TensorDataType) { + return CollectionType.SINGLE; + } else if (fieldType instanceof PrimitiveDataType) { + return CollectionType.SINGLE; + } else if (fieldType instanceof NewDocumentReferenceDataType) { + return CollectionType.SINGLE; + } else { + throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType"); + } + } + + private static Optional<TensorType> convertTensorType(DataType fieldType) { + if ( ! ( fieldType instanceof TensorDataType)) return Optional.empty(); + return Optional.of(((TensorDataType)fieldType).getTensorType()); + } + + private static Optional<StructuredDataType> convertTargetType(DataType fieldType) { + return Optional.of(fieldType) + .filter(NewDocumentReferenceDataType.class::isInstance) + .map(NewDocumentReferenceDataType.class::cast) + .map(NewDocumentReferenceDataType::getTargetType); + } + + /** Converts to the right field type from an attribute type */ + private DataType toDataType(Type attributeType) { + switch (attributeType) { + case STRING : return DataType.STRING; + case INTEGER: return DataType.INT; + case LONG: return DataType.LONG; + case FLOAT16: return DataType.FLOAT16; + case FLOAT: return DataType.FLOAT; + case DOUBLE: return DataType.DOUBLE; + case BOOL: return DataType.BOOL; + case BYTE: return DataType.BYTE; + case PREDICATE: return DataType.PREDICATE; + case TENSOR: return DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new)); + case REFERENCE: return createReferenceDataType(); + default: throw new IllegalArgumentException("Unknown attribute type " + attributeType); + } + } + + @SuppressWarnings("deprecation") + private DataType createReferenceDataType() { + if (referenceDocumentType.isEmpty()) { + throw new IllegalStateException("Referenced document type is not set"); + } + StructuredDataType type = referenceDocumentType.get(); + if (type instanceof DocumentType) { + return new NewDocumentReferenceDataType((DocumentType) type); + } else { + return NewDocumentReferenceDataType.forDocumentName(type.getName()); + } + } + + public DataType getDataType() { + DataType dataType = toDataType(type); + if (collectionType == Attribute.CollectionType.ARRAY) { + return DataType.getArray(dataType); + } else if (collectionType == Attribute.CollectionType.WEIGHTEDSET) { + return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero); + } else { + return dataType; + } + } + + @Override + public int hashCode() { + return Objects.hash( + name, type, collectionType, sorting, dictionary, isPrefetch(), fastAccess, removeIfZero, + createIfNonExistent, isPosition, huge, mutable, paged, enableBitVectors, enableOnlyBitVector, + tensorType, referenceDocumentType, distanceMetric, hnswIndexParams); + } + + @Override + public boolean equals(Object object) { + if (! (object instanceof Attribute)) return false; + + Attribute other = (Attribute)object; + if (!this.name.equals(other.name)) return false; + return isCompatible(other); + } + + /** Returns whether these attributes describes the same entity, even if they have different names */ + public boolean isCompatible(Attribute other) { + if (! this.type.equals(other.type)) return false; + if (! this.collectionType.equals(other.collectionType)) return false; + if (this.isPrefetch() != other.isPrefetch()) return false; + if (this.removeIfZero != other.removeIfZero) return false; + if (this.createIfNonExistent != other.createIfNonExistent) return false; + if (this.enableBitVectors != other.enableBitVectors) return false; + if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false; + if (this.fastSearch != other.fastSearch) return false; + if (this.huge != other.huge) return false; + if (this.mutable != other.mutable) return false; + if (this.paged != other.paged) return false; + if (! this.sorting.equals(other.sorting)) return false; + if (! Objects.equals(dictionary, other.dictionary)) return false; + if (! Objects.equals(tensorType, other.tensorType)) return false; + if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; + if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; + if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false; + + return true; + } + + @Override + public Attribute clone() { + try { + return (Attribute)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + @Override + public String toString() { + return "attribute '" + name + "' (" + type + ")"; + } + + public Set<String> getAliases() { + return aliases; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java new file mode 100644 index 00000000000..8563d414c40 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java @@ -0,0 +1,84 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; + +/** + * Encapsulates values required for predicate fields. + * + * @author lesters + */ +public final class BooleanIndexDefinition { + + public static final int DEFAULT_ARITY = 8; + public static final long DEFAULT_UPPER_BOUND = Long.MAX_VALUE; + public static final long DEFAULT_LOWER_BOUND = Long.MIN_VALUE; + public static final double DEFAULT_DENSE_POSTING_LIST_THRESHOLD = 0.4; + + private final OptionalInt arity; // mandatory field value + private final OptionalLong lowerBound; + private final OptionalLong upperBound; + private final OptionalDouble densePostingListThreshold; + + public BooleanIndexDefinition(Optional<Integer> arity, + Optional<Long> lowerBound, + Optional<Long> upperBound, + Optional<Double> densePLT) + { + this.arity = arity.isPresent() ? OptionalInt.of(arity.get()) : OptionalInt.empty(); + this.lowerBound = lowerBound.isPresent() ? OptionalLong.of(lowerBound.get()) : OptionalLong.empty(); + this.upperBound = upperBound.isPresent() ? OptionalLong.of(upperBound.get()) : OptionalLong.empty(); + this.densePostingListThreshold = densePLT.isPresent() ? OptionalDouble.of(densePLT.get()) : OptionalDouble.empty(); + } + + public BooleanIndexDefinition(OptionalInt arity, OptionalLong lowerBound, + OptionalLong upperBound, OptionalDouble densePostingListThreshold) { + this.arity = arity; + this.lowerBound = lowerBound; + this.upperBound = upperBound; + this.densePostingListThreshold = densePostingListThreshold; + } + + public int getArity() { + return arity.getAsInt(); + } + + public boolean hasArity() { + return arity.isPresent(); + } + + public long getLowerBound() { + return lowerBound.orElse(DEFAULT_LOWER_BOUND); + } + + public boolean hasLowerBound() { + return lowerBound.isPresent(); + } + + public long getUpperBound() { + return upperBound.orElse(DEFAULT_UPPER_BOUND); + } + + public boolean hasUpperBound() { + return upperBound.isPresent(); + } + + public double getDensePostingListThreshold() { + return densePostingListThreshold.orElse(DEFAULT_DENSE_POSTING_LIST_THRESHOLD); + } + + public boolean hasDensePostingListThreshold() { + return densePostingListThreshold.isPresent(); + } + + @Override + public String toString() { + return "BooleanIndexDefinition [arity=" + arity + ", lowerBound=" + + lowerBound + ", upperBound=" + upperBound + ", densePostingListThreshold=" + + densePostingListThreshold + "]"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Case.java b/config-model/src/main/java/com/yahoo/schema/document/Case.java new file mode 100644 index 00000000000..443f67940a0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Case.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * Describes if items should preserve lower/upper case, or shall be uncased + * which normally mean they are all normalized to lowercase. + * @author baldersheim + */ +public enum Case { + CASED("cased"), + UNCASED("uncased"); + private String name; + Case(String name) { this.name = name; } + public String getName() { return name;} +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java new file mode 100644 index 00000000000..993bf16405a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java @@ -0,0 +1,123 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; + +/** + * Utils used to check whether a complex field supports being represented as struct field attributes. + * + * Currently we support: + * - array of simple struct + * - map of primitive type to simple struct + * - map of primitive type to primitive type + * + * A simple struct can contain fields of any type, but only fields of primitive type can be defined as + * struct field attributes in the complex field using the simple struct. + * + * @author geirst + */ +public class ComplexAttributeFieldUtils { + + public static boolean isSupportedComplexField(ImmutableSDField field) { + return (isArrayOfSimpleStruct(field) || + isMapOfSimpleStruct(field) || + isMapOfPrimitiveType(field)); + } + + public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { + if (field.getDataType() instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)field.getDataType(); + return isStructWithPrimitiveStructFieldAttributes(arrayType.getNestedType(), field); + } else { + return false; + } + } + + public static boolean isMapOfSimpleStruct(ImmutableSDField field) { + if (field.getDataType() instanceof MapDataType) { + MapDataType mapType = (MapDataType)field.getDataType(); + return isPrimitiveType(mapType.getKeyType()) && + isStructWithPrimitiveStructFieldAttributes(mapType.getValueType(), + field.getStructField("value")); + } else { + return false; + } + } + + public static boolean isMapOfPrimitiveType(ImmutableSDField field) { + if (field.getDataType() instanceof MapDataType) { + MapDataType mapType = (MapDataType)field.getDataType(); + return isPrimitiveType(mapType.getKeyType()) && + isPrimitiveType(mapType.getValueType()); + } else { + return false; + } + } + + private static boolean isStructWithPrimitiveStructFieldAttributes(DataType type, ImmutableSDField field) { + if (type instanceof StructDataType && ! GeoPos.isPos(type)) { + for (ImmutableSDField structField : field.getStructFields()) { + Attribute attribute = structField.getAttributes().get(structField.getName()); + if (attribute != null) { + if (!isPrimitiveType(attribute)) { + return false; + } + } else if (structField.wasConfiguredToDoAttributing()) { + if (!isPrimitiveType(structField.getDataType())) { + return false; + } + } + } + return true; + } else { + return false; + } + } + + public static boolean isPrimitiveType(Attribute attribute) { + return attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) && + isPrimitiveType(attribute.getDataType()); + } + + public static boolean isPrimitiveType(DataType dataType) { + return dataType.equals(DataType.BYTE) || + dataType.equals(DataType.INT) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.STRING); + } + + public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { + if (isArrayOfSimpleStruct(field)) { + return hasOnlyStructFieldAttributes(field); + } else if (isMapOfSimpleStruct(field)) { + return hasSingleAttribute(field.getStructField("key")) && + hasOnlyStructFieldAttributes(field.getStructField("value")); + } else if (isMapOfPrimitiveType(field)) { + return hasSingleAttribute(field.getStructField("key")) && + hasSingleAttribute(field.getStructField("value")); + } + return false; + } + + private static boolean hasOnlyStructFieldAttributes(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + if (!hasSingleAttribute(structField)) { + return false; + } + } + return true; + } + + private static boolean hasSingleAttribute(ImmutableSDField field) { + if (field.getAttributes().size() != 1) { + return false; + } + return (field.getAttributes().get(field.getName()) != null); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java b/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java new file mode 100644 index 00000000000..4744547c778 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.document; + +/** + * Represents settings for dictionary control + * + * @author baldersheim + */ +public class Dictionary { + public enum Type { BTREE, HASH, BTREE_AND_HASH }; + private Type type = null; + private Case casing= null; + + public void updateType(Type type) { + if (this.type == null) { + this.type = type; + } else if ((this.type == Type.BTREE) && (type == Type.HASH)) { + this.type = Type.BTREE_AND_HASH; + } else if ((this.type == Type.HASH) && (type == Type.BTREE)) { + this.type = Type.BTREE_AND_HASH; + } else { + throw new IllegalArgumentException("Can not combine previous dictionary setting " + this.type + + " with current " + type); + } + } + public void updateMatch(Case casing) { + if (this.casing != null) { + throw new IllegalArgumentException("dictionary match mode has already been set to " + this.casing); + } + this.casing = casing; + } + public Type getType() { return (type != null) ? type : Type.BTREE; } + public Case getMatch() { return (casing != null) ? casing : Case.UNCASED; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java b/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java new file mode 100644 index 00000000000..e62e784d7b3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.TreeSet; + +import static java.util.Comparator.comparing; + +/** + * Searchable collection of fields. + * + * @author baldersheim + */ +public class FieldSet { + + private final String name; + private final Set<String> queryCommands = new LinkedHashSet<>(); + private final Set<String> fieldNames = new TreeSet<>(); + private final Set<ImmutableSDField> fields = new TreeSet<>(comparing(ImmutableSDField::asField)); + private Matching matching = null; + + public FieldSet(String name) { this.name = name; } + public String getName() { return name; } + public FieldSet addFieldName(String field) { fieldNames.add(field); return this; } + public Set<String> getFieldNames() { return fieldNames; } + public Set<ImmutableSDField> fields() { return fields; } + public Set<String> queryCommands() { return queryCommands; } + + public void setMatching(Matching matching) { + this.matching = matching; + } + + public Matching getMatching() { + return matching; + } + + @Override + public String toString() { return "fieldset '" + name + "'"; } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java b/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java new file mode 100644 index 00000000000..829555d88c6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; + +/** + * Common utilities for recognizing fields with the built-in "position" datatype, + * possibly in array form. + * @author arnej + */ +public class GeoPos { + static public boolean isPos(DataType type) { + return PositionDataType.INSTANCE.equals(type); + } + static public boolean isPosArray(DataType type) { + return DataType.getArray(PositionDataType.INSTANCE).equals(type); + } + static public boolean isAnyPos(DataType type) { + return isPos(type) || isPosArray(type); + } + + static public boolean isPos(ImmutableSDField field) { return isPos(field.getDataType()); } + static public boolean isPosArray(ImmutableSDField field) { return isPosArray(field.getDataType()); } + static public boolean isAnyPos(ImmutableSDField field) { return isAnyPos(field.getDataType()); } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java new file mode 100644 index 00000000000..cc427356c78 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Optional; + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. + * + * @author geirst + */ +public class HnswIndexParams { + + public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; + public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; + + private final Optional<Integer> maxLinksPerNode; + private final Optional<Integer> neighborsToExploreAtInsert; + private final Optional<Boolean> multiThreadedIndexing; + + public static class Builder { + private Optional<Integer> maxLinksPerNode = Optional.empty(); + private Optional<Integer> neighborsToExploreAtInsert = Optional.empty(); + private Optional<Boolean> multiThreadedIndexing = Optional.empty(); + + public void setMaxLinksPerNode(int value) { + maxLinksPerNode = Optional.of(value); + } + public void setNeighborsToExploreAtInsert(int value) { + neighborsToExploreAtInsert = Optional.of(value); + } + public void setMultiThreadedIndexing(boolean value) { + multiThreadedIndexing = Optional.of(value); + } + public HnswIndexParams build() { + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert, multiThreadedIndexing); + } + } + + public HnswIndexParams() { + this.maxLinksPerNode = Optional.empty(); + this.neighborsToExploreAtInsert = Optional.empty(); + this.multiThreadedIndexing = Optional.empty(); + } + + public HnswIndexParams(Optional<Integer> maxLinksPerNode, + Optional<Integer> neighborsToExploreAtInsert, + Optional<Boolean> multiThreadedIndexing) { + this.maxLinksPerNode = maxLinksPerNode; + this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; + this.multiThreadedIndexing = multiThreadedIndexing; + } + + /** + * Creates a new instance where values from the given parameter instance are used where they are present, + * otherwise we use values from this. + */ + public HnswIndexParams overrideFrom(Optional<HnswIndexParams> other) { + if (! other.isPresent()) return this; + HnswIndexParams rhs = other.get(); + return new HnswIndexParams(rhs.maxLinksPerNode.or(() -> maxLinksPerNode), + rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert), + rhs.multiThreadedIndexing.or(() -> multiThreadedIndexing)); + } + + public int maxLinksPerNode() { + return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE); + } + + public int neighborsToExploreAtInsert() { + return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); + } + + public boolean multiThreadedIndexing() { + return multiThreadedIndexing.orElse(true); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java new file mode 100644 index 00000000000..553b5b4d940 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Collection; + +import static java.util.stream.Collectors.toList; + +/** + * Wraps {@link ImportedComplexField} as {@link ImmutableSDField}. + */ +public class ImmutableImportedComplexSDField extends ImmutableImportedSDField { + private final ImportedComplexField importedComplexField; + + public ImmutableImportedComplexSDField(ImportedComplexField importedField) { + super(importedField); + importedComplexField = importedField; + } + + @Override + public ImmutableSDField getStructField(String name) { + ImportedField field = importedComplexField.getNestedField(name); + return (field != null) ? field.asImmutableSDField() : null; + } + + @Override + public Collection<? extends ImmutableSDField> getStructFields() { + return importedComplexField.getNestedFields().stream().map(field -> field.asImmutableSDField()).collect(toList()); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java new file mode 100644 index 00000000000..335942de99d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java @@ -0,0 +1,218 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Wraps {@link ImportedField} as {@link ImmutableSDField}. + * Methods that are not meaningful or relevant for imported fields will throw {@link UnsupportedOperationException}. + * + * @author bjorncs + */ +public class ImmutableImportedSDField implements ImmutableSDField { + + private final ImportedField importedField; + + ImmutableImportedSDField(ImportedField importedField) { + this.importedField = importedField; + } + + public ImportedField getImportedField() { + return importedField; + } + + @Override + public <T extends Expression> boolean containsExpression(Class<T> searchFor) { + throw createUnsupportedException(searchFor.getSimpleName()); + } + + @Override + public boolean doesAttributing() { + return importedField.targetField().doesAttributing(); + } + + @Override + public boolean doesIndexing() { + return importedField.targetField().doesIndexing(); + } + + @Override + public boolean doesLowerCasing() { + return importedField.targetField().doesLowerCasing(); + } + + @Override + public boolean isExtraField() { + return false; + } + + @Override + public boolean isImportedField() { + return true; + } + + @Override + public boolean isIndexStructureField() { + return importedField.targetField().isIndexStructureField(); + } + + @Override + public boolean hasIndex() { + return importedField.targetField().hasIndex(); + } + + @Override + public boolean usesStructOrMap() { + return importedField.targetField().usesStructOrMap(); + } + + @Override + public boolean wasConfiguredToDoAttributing() { + return importedField.targetField().wasConfiguredToDoAttributing(); + } + + @Override + public DataType getDataType() { + return importedField.targetField().getDataType(); + } + + @Override + public SummaryField getSummaryField(String name) { + return importedField.targetField().getSummaryField(name); + } + + @Override + public Index getIndex(String name) { + if ( ! importedField.fieldName().equals(name)) { + throw new IllegalArgumentException("Getting an index (" + name + ") with different name than the imported field (" + + importedField.fieldName() + ") is not supported"); + } + String targetIndexName = importedField.targetField().getName(); + return importedField.targetField().getIndex(targetIndexName); + } + + @Override + public List<String> getQueryCommands() { + return importedField.targetField().getQueryCommands(); + } + + @Override + public Map<String, Attribute> getAttributes() { + return importedField.targetField().getAttributes(); + } + + @Override + public Attribute getAttribute() { return importedField.targetField().getAttribute(); } + + @Override + public Map<String, String> getAliasToName() { + return Collections.emptyMap(); + } + + @Override + public ScriptExpression getIndexingScript() { + throw createUnsupportedException("indexing"); + } + + @Override + public Matching getMatching() { + return importedField.targetField().getMatching(); + } + + @Override + public NormalizeLevel getNormalizing() { + return importedField.targetField().getNormalizing(); + } + + @Override + public ImmutableSDField getStructField(String name) { + throw createUnsupportedException("struct"); + } + + @Override + public Collection<? extends ImmutableSDField> getStructFields() { + throw createUnsupportedException("struct"); + } + + @Override + public Stemming getStemming() { + return importedField.targetField().getStemming(); + } + + @Override + public Stemming getStemming(Schema schema) { + throw createUnsupportedException("stemming"); + } + + @Override + public Ranking getRanking() { + throw createUnsupportedException("ranking"); + } + + @Override + public Map<String, SummaryField> getSummaryFields() { + throw createUnsupportedException("summary fields"); + } + + @Override + public String getName() { + return importedField.fieldName(); // Name of the imported field, not the target field + } + + @Override + public int getWeight() { + return importedField.targetField().getWeight(); + } + + @Override + public int getLiteralBoost() { + return importedField.targetField().getLiteralBoost(); + } + + @Override + public RankType getRankType() { + return importedField.targetField().getRankType(); + } + + @Override + public Map<String, Index> getIndices() { + return importedField.targetField().getIndices(); + } + + @Override + public boolean existsIndex(String name) { + return importedField.targetField().existsIndex(name); + } + + /** + * Returns a field representation of the imported field. + * Changes to the returned instance are not propagated back to the underlying imported field! + */ + @Override + public Field asField() { + return new Field( + importedField.fieldName(), + importedField.targetField().getDataType()); + } + + private static UnsupportedOperationException createUnsupportedException(String aspect) { + return new UnsupportedOperationException("'" + aspect + "' is not meaningful or relevant for an imported field."); + } + + @Override + public boolean hasFullIndexingDocprocRights() { + return importedField.targetField().hasFullIndexingDocprocRights(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java new file mode 100644 index 00000000000..44e442811ba --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * An interface containing the non-mutating methods of {@link SDField}. + * For description of the methods see {@link SDField}. + * + * @author bjorncs + */ +public interface ImmutableSDField { + + <T extends Expression> boolean containsExpression(Class<T> searchFor); + + boolean doesAttributing(); + + boolean doesIndexing(); + + boolean doesLowerCasing(); + + boolean isExtraField(); + + boolean isImportedField(); + + boolean isIndexStructureField(); + + boolean usesStructOrMap(); + + /** + * Whether this field at some time was configured to do attributing. + * + * This function can typically return a different value than doesAttributing(), + * which uses the final state of the underlying indexing script instead. + */ + boolean wasConfiguredToDoAttributing(); + + DataType getDataType(); + + Index getIndex(String name); + + List<String> getQueryCommands(); + + Map<String, Attribute> getAttributes(); + + Attribute getAttribute(); + + Map<String, String> getAliasToName(); + + ScriptExpression getIndexingScript(); + + Matching getMatching(); + + NormalizeLevel getNormalizing(); + + ImmutableSDField getStructField(String name); + + Collection<? extends ImmutableSDField> getStructFields(); + + Stemming getStemming(); + + Stemming getStemming(Schema schema); + + Ranking getRanking(); + + String getName(); + + Map<String, SummaryField> getSummaryFields(); + + /** Returns a {@link Field} representation (which is sadly not immutable) */ + Field asField(); + + boolean hasFullIndexingDocprocRights(); + int getWeight(); + int getLiteralBoost(); + RankType getRankType(); + Map<String, Index> getIndices(); + boolean existsIndex(String name); + SummaryField getSummaryField(String name); + boolean hasIndex(); +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java new file mode 100644 index 00000000000..10c400bc4d6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.DocumentReference; + +import java.util.Collection; +import java.util.Map; + +/** + * A complex field that is imported from a concrete field in a referenced document type and given an alias name. + */ +public class ImportedComplexField extends ImportedField { + + private Map<String, ImportedField> nestedFields; + + public ImportedComplexField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { + super(fieldName, reference, targetField); + nestedFields = new java.util.LinkedHashMap<>(0); + } + + @Override + public ImmutableSDField asImmutableSDField() { + return new ImmutableImportedComplexSDField(this); + } + + public void addNestedField(ImportedField importedField) { + String prefix = fieldName() + "."; + assert(importedField.fieldName().substring(0, prefix.length()).equals(prefix)); + String suffix = importedField.fieldName().substring(prefix.length()); + nestedFields.put(suffix, importedField); + } + + public Collection<ImportedField> getNestedFields() { + return nestedFields.values(); + } + + public ImportedField getNestedField(String name) { + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + ImportedField superField = nestedFields.get(superFieldName); + if (superField != null && superField instanceof ImportedComplexField) { + return ((ImportedComplexField)superField).getNestedField(subFieldName); + } + return null; + } + return nestedFields.get(name); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java new file mode 100644 index 00000000000..50f8591bbce --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.DocumentReference; + +/** + * A field that is imported from a concrete field in a referenced document type and given an alias name. + * + * @author geirst + */ +public abstract class ImportedField { + + private final String fieldName; + private final DocumentReference reference; + private final ImmutableSDField targetField; + + public ImportedField(String fieldName, + DocumentReference reference, + ImmutableSDField targetField) { + this.fieldName = fieldName; + this.reference = reference; + this.targetField = targetField; + } + + public String fieldName() { + return fieldName; + } + + public DocumentReference reference() { + return reference; + } + + public ImmutableSDField targetField() { + return targetField; + } + + public abstract ImmutableSDField asImmutableSDField(); +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java new file mode 100644 index 00000000000..f6654896fae --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Collections; +import java.util.Map; + +/** + * A set of fields that are imported from concrete fields in referenced document types. + * + * @author geirst + */ +public class ImportedFields { + + private final Map<String, ImportedField> fields; + + public ImportedFields(Map<String, ImportedField> fields) { + this.fields = fields; + } + + public Map<String, ImportedField> fields() { + return Collections.unmodifiableMap(fields); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java new file mode 100644 index 00000000000..244135ecc10 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.DocumentReference; + +/** + * A simple field that is imported from a concrete field in a referenced document type and given an alias name. + */ +public class ImportedSimpleField extends ImportedField { + public ImportedSimpleField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { + super(fieldName, reference, targetField); + } + + @Override + public ImmutableSDField asImmutableSDField() { + return new ImmutableImportedSDField(this); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java b/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java new file mode 100644 index 00000000000..8556fe491d0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** Which match algorithm is used by this matching setup */ + +public enum MatchAlgorithm { + NORMAL("normal"), + PREFIX("prefix"), + SUBSTRING("substring"), + SUFFIX("suffix"); + + private String name; + MatchAlgorithm(String name) { this.name = name; } + + public String getName() { return name; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/MatchType.java b/config-model/src/main/java/com/yahoo/schema/document/MatchType.java new file mode 100644 index 00000000000..d2088e71282 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/MatchType.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +public enum MatchType { + TEXT("text"), + WORD("word"), + EXACT("exact"), + GRAM("gram"); + + private String name; + MatchType(String name) { this.name = name; } + + public String getName() { return name; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java new file mode 100644 index 00000000000..f70f31be0bd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -0,0 +1,141 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.io.Serializable; + +/** + * Defines how a field should be matched. + * Matching objects can be compared based on their content, but they are <i>not</i> immutable. + * + * @author bratseth + */ +public class Matching implements Cloneable, Serializable { + + public static final MatchType defaultType = MatchType.TEXT; + + private MatchType type = MatchType.TEXT; + private Case casing = Case.UNCASED; + + /** The basic match algorithm */ + private MatchAlgorithm algorithm = MatchAlgorithm.NORMAL; + + private boolean typeUserSet = false; + + private boolean algorithmUserSet = false; + + /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */ + private int gramSize=-1; + + /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ + private Integer maxLength; + + private String exactMatchTerminator=null; + + /** Creates a matching of type "text" */ + public Matching() {} + + public Matching(MatchType type) { + this.type = type; + } + + public MatchType getType() { return type; } + public Case getCase() { return casing; } + + public void setType(MatchType type) { + this.type = type; + typeUserSet = true; + } + + public void setCase(Case casing) { this.casing = casing; } + + public Integer maxLength() { return maxLength; } + public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } + public boolean isTypeUserSet() { return typeUserSet; } + + public MatchAlgorithm getAlgorithm() { return algorithm; } + + public void setAlgorithm(MatchAlgorithm algorithm) { + this.algorithm = algorithm; + algorithmUserSet = true; + } + + public boolean isAlgorithmUserSet() { return algorithmUserSet; } + + public boolean isPrefix() { return algorithm == MatchAlgorithm.PREFIX; } + + public boolean isSubstring() { return algorithm == MatchAlgorithm.SUBSTRING; } + + public boolean isSuffix() { return algorithm == MatchAlgorithm.SUFFIX; } + + /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ + public int getGramSize() { return gramSize; } + + public void setGramSize(int gramSize) { this.gramSize=gramSize; } + + /** + * Merge data from another matching object + */ + public void merge(Matching m) { + if (m == null) return; + if (m.isAlgorithmUserSet()) { + this.setAlgorithm(m.getAlgorithm()); + } + if (m.isTypeUserSet()) { + this.setType(m.getType()); + if (m.getType() == MatchType.GRAM) + gramSize = m.gramSize; + } + if (m.getExactMatchTerminator() != null) { + this.setExactMatchTerminator(m.getExactMatchTerminator()); + } + } + + /** + * If exact matching is used, this returns the terminator string + * which terminates an exact matched sequence in queries. If exact + * matching is not used, or no terminator is set, this is null + */ + public String getExactMatchTerminator() { return exactMatchTerminator; } + + /** + * Sets the terminator string which terminates an exact matched + * sequence in queries (used if type is EXACT). + */ + public void setExactMatchTerminator(String exactMatchTerminator) { + this.exactMatchTerminator = exactMatchTerminator; + } + + @Override + public String toString() { + return type + " matching [" + (type==MatchType.GRAM ? "gram size " + gramSize : "supports " + algorithm) + + "], [exact-terminator "+exactMatchTerminator+"]"; + } + + public Matching clone() { + try { + return (Matching)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + @Override + public boolean equals(Object o) { + if (! (o instanceof Matching)) return false; + + Matching other=(Matching)o; + if ( ! other.type.equals(this.type)) return false; + if ( ! other.algorithm.equals(this.algorithm)) return false; + if ( this.exactMatchTerminator == null && other.exactMatchTerminator != null) return false; + if ( this.exactMatchTerminator != null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) + return false; + if ( gramSize != other.gramSize) return false; + return true; + } + + @Override public int hashCode() { + return java.util.Objects.hash(type, algorithm, exactMatchTerminator, gramSize); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java b/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java new file mode 100644 index 00000000000..12880d0e1c8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * class representing the character normalization + * we want to do on query and indexed text. + * Levels are strict subsets, so doing accent + * removal means doing codepoint normalizing + * and case normalizing also. + */ +// TODO: Missing author +public class NormalizeLevel { + + /** + * The current levels are as follows: + * NONE: no changes to input text + * CODEPOINT: convert text into Unicode + * Normalization Form Compatibility Composition + * LOWERCASE: also convert text into lowercase letters + * ACCENT: do both above and remove accents on characters + */ + public enum Level { + NONE, CODEPOINT, LOWERCASE, ACCENT + } + + private boolean userSpecified = false; + private Level level = Level.ACCENT; + + /** + * Returns whether accents should be removed from text + */ + public boolean doRemoveAccents() { return level == Level.ACCENT; } + + /** + * Construct a default (full) normalizelevel, + */ + public NormalizeLevel() {} + + /** + * Construct for a specific level, possibly user specified + * + * @param level which level to use + * @param fromUser whether this was specified by the user + */ + public NormalizeLevel(Level level, boolean fromUser) { + this.level = level; + this.userSpecified = fromUser; + } + + /** + * Change the current level to CODEPOINT as inferred + * by other features' needs. If the current level + * was user specified it will not change; also this + * will not increase the level. + */ + public void inferCodepoint() { + if (userSpecified) { + // ignore inferred changes if user specified something + return; + } + // do not increase level + if (level != Level.NONE) level = Level.CODEPOINT; + } + + /** + * Change the current level to LOWERCASE as inferred + * by other features' needs. If the current level + * was user specified it will not change; also this + * will not increase the level. + */ + public void inferLowercase() { + if (userSpecified) { + // ignore inferred changes if user specified something + return; + } + // do not increase level + if (level == Level.NONE) return; + if (level == Level.CODEPOINT) return; + + level = Level.LOWERCASE; + } + + public Level getLevel() { + return level; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/RankType.java b/config-model/src/main/java/com/yahoo/schema/document/RankType.java new file mode 100644 index 00000000000..067c1e7f266 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/RankType.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * The rank type of a field. For now this is just a container of a string name. + * This class is immutable. + * + * @author bratseth + */ +public enum RankType { + + /** *implicit* default: No type has been set. */ + DEFAULT, + + // Rank types which can be set explicitly. These are defined for Vespa in NativeRankTypeDefinitionSet + IDENTITY, ABOUT, TAGS, EMPTY; + + @Override + public String toString() { + return "rank type " + name().toLowerCase(); + } + + /** + * Returns the rank type from a string, regardless of its case. + * + * @param rankTypeName a rank type name in any casing + * @return the rank type found + * @throws IllegalArgumentException if not found + */ + public static RankType fromString(String rankTypeName) { + try { + return RankType.valueOf(rankTypeName.toUpperCase()); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown rank type '" + rankTypeName + "'. Supported rank types are " + + "'identity', 'about', 'tags' and 'empty'."); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Ranking.java b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java new file mode 100644 index 00000000000..31fd9747e2d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.io.Serializable; + +/** + * The rank settings given in a rank clause in the search definition. + * + * @author Vegard Havdal + */ +public class Ranking implements Cloneable, Serializable { + + private boolean literal = false; + private boolean filter = false; + private boolean normal = false; + + /** + * <p>Returns whether literal (non-stemmed, non-normalized) forms of the words should + * be indexed in a separate index which is searched by a automatically added rank term + * during searches.</p> + * + * <p>Default is false.</p> + */ + public boolean isLiteral() { return literal; } + + public void setLiteral(boolean literal) { this.literal = literal; } + + /** + * <p>Returns whether this is a filter. Filters will only tell if they are matched or not, + * no detailed relevance information will be available about the match.</p> + * + * <p>Matching a filter is much cheaper for the search engine than matching a regular field.</p> + * + * <p>Default is false.</p> + */ + public boolean isFilter() { return filter && !normal; } + + public void setFilter(boolean filter) { this.filter = filter; } + + /** Whether user has explicitly requested normal (non-filter) behavior */ + public boolean isNormal() { return normal; } + public void setNormal(boolean n) { this.normal = n; } + + /** Returns true if the given rank settings are the same */ + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Ranking)) return false; + + Ranking other=(Ranking)o; + if (this.filter != other.filter) return false; + if (this.literal != other.literal) return false; + if (this.normal != other.normal) return false; + return true; + } + + @Override + public int hashCode() { + return java.util.Objects.hash(filter, literal, normal); + } + + @Override + public String toString() { + return "rank settings [filter: " + filter + ", literal: " + literal + ", normal: "+normal+"]"; + } + + @Override + public Ranking clone() { + try { + return (Ranking)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error", e); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java new file mode 100644 index 00000000000..d300bd08bfd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -0,0 +1,347 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.AnnotationTypeRegistry; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.FieldSets; +import com.yahoo.schema.Schema; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * A document definition is a list of fields. Documents may inherit other documents, + * implicitly acquiring their fields as it's own. If a document is not set to inherit + * any document, it will always inherit the document "document.0". + * + * @author Thomas Gundersen + * @author bratseth + */ +public class SDDocumentType implements Cloneable, Serializable { + + public static final SDDocumentType VESPA_DOCUMENT; + private final Map<DataTypeName, SDDocumentType> inheritedTypes = new LinkedHashMap<>(); + private final Map<NewDocumentType.Name, SDDocumentType> ownedTypes = new LinkedHashMap<>(); + private final AnnotationTypeRegistry annotationTypes = new AnnotationTypeRegistry(); + private DocumentType docType; + private DataType structType; + // The field sets here are set from the processing step in SD, + // to ensure that the full Search and this SDDocumentType is built first. + private FieldSets fieldSets; + // Document references + private Optional<DocumentReferences> documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; + + static { + VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); + VESPA_DOCUMENT.addType(createSDDocumentType(PositionDataType.INSTANCE)); + } + + public SDDocumentType clone() throws CloneNotSupportedException { + SDDocumentType type = (SDDocumentType) super.clone(); + type.docType = docType.clone(); + type.inheritedTypes.putAll(inheritedTypes); + type.structType = structType; + // TODO this isn't complete; should it be..?! + return type; + } + + /** + * For adding structs defined in document scope + * + * @param dt the struct to add + * @return self, for chaining + */ + public SDDocumentType addType(SDDocumentType dt) { + NewDocumentType.Name name = new NewDocumentType.Name(dt.getName()); + if (getType(name) != null) + throw new IllegalArgumentException("Data type '" + name + "' has already been used."); + if (name.getName() == docType.getName()) + throw new IllegalArgumentException("Data type '" + name + "' can not have same name as its defining document."); + ownedTypes.put(name, dt); + return this; + } + public final SDDocumentType getOwnedType(String name) { + return getOwnedType(new NewDocumentType.Name(name)); + } + public SDDocumentType getOwnedType(DataTypeName name) { + return getOwnedType(name.getName()); + } + + public SDDocumentType getOwnedType(NewDocumentType.Name name) { + return ownedTypes.get(name); + } + + public final SDDocumentType getType(String name) { + return getType(new NewDocumentType.Name(name)); + } + + public SDDocumentType getType(NewDocumentType.Name name) { + SDDocumentType type = ownedTypes.get(name); + if (type == null) { + for (SDDocumentType inherited : inheritedTypes.values()) { + type = inherited.getType(name); + if (type != null) { + return type; + } + } + } + return type; + } + + public SDDocumentType addAnnotation(AnnotationType annotation) { + annotationTypes.register(annotation); + return this; + } + + /** Returns all owned datatypes. */ + public Collection<SDDocumentType> getTypes() { return ownedTypes.values(); } + + // TODO: Include inherited + public Map<String, AnnotationType> getAnnotations() { return annotationTypes.getTypes(); } + public AnnotationType findAnnotation(String name) { return annotationTypes.getType(name); } + + public Collection<SDDocumentType> getAllTypes() { + Collection<SDDocumentType> list = new ArrayList<>(); + list.addAll(getTypes()); + for (SDDocumentType inherited : inheritedTypes.values()) { + list.addAll(inherited.getAllTypes()); + } + return list; + } + + public Map<NewDocumentType.Name, SDDocumentType> allTypes() { + Map<NewDocumentType.Name, SDDocumentType> map = new LinkedHashMap<>(); + for (SDDocumentType inherited : inheritedTypes.values()) + map.putAll(inherited.allTypes()); + map.putAll(ownedTypes); + return map; + } + + /** + * Creates a new document type. + * The document type id will be generated as a hash from the document type name. + * + * @param name The name of the new document type + */ + public SDDocumentType(String name) { + this(name,null); + } + + public SDDocumentType(DataTypeName name) { + this(name.getName()); + } + + /** + * Creates a new document type. + * The document type id will be generated as a hash from the document type name. + * + * @param name the name of the new document type + * @param schema check for type ID collisions in this search definition + */ + public SDDocumentType(String name, Schema schema) { + docType = new DocumentType(name); + validateId(schema); + inherit(VESPA_DOCUMENT); + } + + public boolean isStruct() { return getStruct() != null; } + public DataType getStruct() { return structType; } + public SDDocumentType setStruct(DataType structType) { + if (structType != null) { + this.structType = structType; + inheritedTypes.remove(VESPA_DOCUMENT.getDocumentName()); + } else { + if (docType.contentStruct() != null) { + this.structType = docType.contentStruct(); + inheritedTypes.clear(); + } else { + throw new IllegalArgumentException("You can not set a null struct"); + } + } + return this; + } + + public String getName() { return docType.getName(); } + public DataTypeName getDocumentName() { return docType.getDataTypeName(); } + public DocumentType getDocumentType() { return docType; } + + public void inherit(DataTypeName name) { + inherit(new TemporarySDDocumentType(name)); + } + + public void inherit(SDDocumentType type) { + if (type == null) return; + if (type.getName().equals(this.getName())) + throw new IllegalArgumentException("Document type '" + getName() + "' cannot inherit itself"); + if ( ! inheritedTypes.containsKey(type.getDocumentName()) || + (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) { + inheritedTypes.put(type.getDocumentName(), type); + } + } + + public Collection<SDDocumentType> getInheritedTypes() { return inheritedTypes.values(); } + + public Map<DataTypeName, SDDocumentType> inheritedTypes() { return inheritedTypes; } + + protected void validateId(Schema schema) { + if (schema == null) return; + if (schema.getDocument(getName()) == null) return; + SDDocumentType doc = schema.getDocument(); + throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " + + "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); + } + + public void setFieldId(SDField field, int id) { + field.setId(id, docType); + } + + /** Override getField, as it may need to ask inherited types that isn't registered in document type. */ + public Field getField(String name) { + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + Field f = docType.getField(superFieldName); + if (f != null) { + if (f instanceof SDField) { + SDField superField = (SDField)f; + return superField.getStructField(subFieldName); + } else { + throw new IllegalArgumentException("Field " + f.getName() + " is not an SDField"); + } + } + } + Field f = docType.getField(name); + if (f == null) { + for(SDDocumentType parent : inheritedTypes.values()) { + f = parent.getField(name); + if (f != null) return f; + } + } + return f; + } + + public void addField(Field field) { + verifyInheritance(field); + for (Iterator<Field> i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) { + if (field.getName().equalsIgnoreCase((i.next()).getName())) { + throw new IllegalArgumentException("Duplicate (case insensitively) " + field + " in " + this); + } + } + docType.addField(field); + } + + /** Parse-time inheritance check. */ + private void verifyInheritance(Field field) { + for (SDDocumentType parent : inheritedTypes.values()) { + for (Field pField : parent.fieldSet()) { + if (pField.getName().equals(field.getName())) { + if (!pField.getDataType().equals(field.getDataType())) { + throw new IllegalArgumentException("For " + this + ", field '" + field.getName() + + "': Datatype can not be different from that of same field " + + "in the supertype '" + parent.getName() + "'"); + } + } + } + } + } + + public SDField addField(String string, DataType dataType) { + SDField field = new SDField(this, string, dataType); + addField(field); + return field; + } + + public Field addField(String fName, DataType dataType, boolean header, int code) { + SDField field = new SDField(this, fName, code, dataType); + addField(field); + return field; + } + + private Map<String, Field> fieldsInherited() { + Map<String, Field> map = new LinkedHashMap<>(); + for (SDDocumentType parent : inheritedTypes.values()) { + for (Field field : parent.fieldSet()) { + map.put(field.getName(), field); + } + } + return map; + } + + public Set<Field> fieldSet() { + Map<String, Field> map = fieldsInherited(); + Iterator<Field> it = docType.fieldIteratorThisTypeOnly(); + while (it.hasNext()) { + Field field = it.next(); + map.put(field.getName(), field); + } + return new LinkedHashSet<>(map.values()); + } + + public Iterator<Field> fieldIterator() { + return fieldSet().iterator(); + } + + /** Returns the number of fields in this only, not including inherited fields */ + // TODO: Remove + public int getFieldCount() { + return docType.getFieldCount(); + } + + @Override + public String toString() { + return "document type '" + docType.getName() + "'"; + } + + private static SDDocumentType createSDDocumentType(StructDataType structType) { + SDDocumentType docType = new SDDocumentType(structType.getName()); + for (Field field : structType.getFields()) { + docType.addField(new SDField(docType, field.getName(), field.getDataType())); + } + docType.setStruct(structType); + return docType; + } + + /** The field sets defined for this type and its {@link Schema} */ + public FieldSets getFieldSets() { + return fieldSets; + } + + /** Sets the field sets for this */ + public void setFieldSets(FieldSets fieldSets) { + this.fieldSets = fieldSets; + } + + public Optional<DocumentReferences> getDocumentReferences() { + return documentReferences; + } + + public void setDocumentReferences(DocumentReferences documentReferences) { + this.documentReferences = Optional.of(documentReferences); + } + + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java new file mode 100644 index 00000000000..668b6388620 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -0,0 +1,802 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.documentmodel.OwnedTemporaryType; +import com.yahoo.documentmodel.TemporaryUnknownType; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.fieldoperation.FieldOperation; +import com.yahoo.schema.fieldoperation.FieldOperationContainer; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.TreeMap; + + +/** + * The field class represents a document field. It is used in + * the Document class to get and set fields. Each SDField has a name, a numeric ID, + * a data type. The numeric ID is used when the fields are stored + * in serialized form. + * + * @author bratseth + */ +public class SDField extends Field implements TypedKey, FieldOperationContainer, ImmutableSDField { + + /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ + private boolean indexStructureField = false; + + /** The indexing statements to be applied to this value during indexing */ + private ScriptExpression indexingScript = new ScriptExpression(); + + /** The default rank type for indices of this field */ + private RankType rankType = RankType.DEFAULT; + + /** Rank settings in a "rank" block for the field. */ + private final Ranking ranking = new Ranking(); + + /** + * The literal boost of this field. This boost is added to a rank score + * when a query term matched as query term exactly (unnormalized and unstemmed). + * Non-positive boosts causes no boosting, 0 allows boosts + * to be specified in other rank profiles, while negative values + * turns the capability off. + */ + private int literalBoost = -1; + + /** + * The weight of this field. This is a percentage, + * so 100 is default to provide the identity transform. + */ + private int weight = 100; + + /** + * Indicates what kind of matching should be done on this field + */ + private Matching matching = new Matching(); + + private Dictionary dictionary = null; + + /** Attribute settings, or null if there are none */ + private final Map<String, Attribute> attributes = new TreeMap<>(); + + /** + * The stemming setting of this field, or null to use the default. + * Default is determined by the owning search definition. + */ + private Stemming stemming = null; + + /** How content of this field should be accent normalized etc. */ + private NormalizeLevel normalizing = new NormalizeLevel(); + + /** Extra query commands of this field */ + private final List<String> queryCommands = new java.util.ArrayList<>(0); + + /** Summary fields defined in this field */ + private final Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>(0); + + /** The explicitly index settings on this field */ + private final Map<String, Index> indices = new java.util.LinkedHashMap<>(); + + private boolean idOverride = false; + + /** Struct fields defined in this field */ + private final Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0); + + /** The document that this field was declared in, or null */ + private SDDocumentType repoDocType = null; + + /** The aliases declared for this field. May pertain to indexes or attributes */ + private final Map<String, String> aliasToName = new HashMap<>(); + + /** Pending operations that must be applied after parsing, due to use of not-yet-defined structs. */ + private final List<FieldOperation> pendingOperations = new LinkedList<>(); + + private boolean isExtraField = false; + + private boolean wasConfiguredToDoAttributing = false; + + /** + * Creates a new field. This method is only used to create reserved fields. + * + * @param name the name of the field + * @param dataType the datatype of the field + */ + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { + super(name, id, dataType); + this.repoDocType = repo; + populate(name, dataType); + } + + public SDField(String name, DataType dataType) { + this(null, name, dataType); + } + + /** Creates a new field */ + public SDField(SDDocumentType repo, String name, DataType dataType) { + this(repo, name, dataType, null); + } + + /** Creates a new field */ + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + this(repo, name, dataType, owner, null, 0); + } + + /** + * Creates a new field + * + * @param name the name of the field + * @param dataType the datatype of the field + * @param owner the owning document (used to check for id collisions) + * @param fieldMatching the matching object to set for the field + */ + protected SDField(SDDocumentType repo, + String name, + DataType dataType, + SDDocumentType owner, + Matching fieldMatching, + int recursion) + { + super(name, dataType, owner == null ? null : owner.getDocumentType()); + this.repoDocType = repo; + this.structFieldDepth = recursion; + if (fieldMatching != null) + this.setMatching(fieldMatching); + populate(name, dataType); + } + + private int structFieldDepth = 0; + + private void populate(String name, DataType dataType) { + if (dataType instanceof TensorDataType) { + TensorType type = ((TensorDataType)dataType).getTensorType(); + if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + name + " type " + type + + ": Dense tensor dimensions must have a size"); + addQueryCommand("type " + type); + } + else if (dataType instanceof WeightedSetDataType) { + var nested = ((WeightedSetDataType) dataType).getNestedType().getName(); + addQueryCommand("type WeightedSet<" + nested + ">"); + } + else { + addQueryCommand("type " + dataType.getName()); + } + } + + public void setIsExtraField(boolean isExtra) { + isExtraField = isExtra; + } + + @Override + public boolean isExtraField() { + return isExtraField; + } + + @Override + public boolean isImportedField() { + return false; + } + + @Override + public boolean doesAttributing() { + return containsExpression(AttributeExpression.class); + } + + @Override + public boolean doesIndexing() { + return containsExpression(IndexExpression.class); + } + + public boolean doesSummarying() { + if (usesStruct()) { + for (SDField structField : getStructFields()) { + if (structField.doesSummarying()) { + return true; + } + } + } + return containsExpression(SummaryExpression.class); + } + + @Override + public boolean doesLowerCasing() { + return containsExpression(LowerCaseExpression.class); + } + + @Override + public <T extends Expression> boolean containsExpression(Class<T> searchFor) { + return findExpression(searchFor) != null; + } + + private <T extends Expression> T findExpression(Class<T> searchFor) { + return new ExpressionSearcher<>(searchFor).searchIn(indexingScript); + } + + public void addSummaryFieldSources(SummaryField summaryField) { + if (usesStruct()) { + /* + * How this works for structs: When at least one sub-field in a struct is to + * be used for summary, that whole struct field is included in summary.cfg. Then, + * vsmsummary.cfg specifies the sub-fields used for each struct field. + * So we recurse into each struct, adding the destination classes set for each sub-field + * to the main summary-field for the struct field. + */ + for (SDField structField : getStructFields()) { + for (SummaryField sumF : structField.getSummaryFields().values()) { + for (String dest : sumF.getDestinations()) { + summaryField.addDestination(dest); + } + } + structField.addSummaryFieldSources(summaryField); + } + } else { + if (doesSummarying()) { + summaryField.addSource(getName()); + } + } + } + + private boolean doneStructFields = false; + + @SuppressWarnings("deprecation") + private void actuallyMakeStructFields() { + if (doneStructFields) return; + if (getFirstStructOrMapRecursive() == null) { + doneStructFields = true; + return; + } + var sdoc = repoDocType; + var dataType = getDataType(); + + java.util.function.BiConsumer<String, DataType> supplyStructField = (fieldName, fieldType) -> { + if (structFields.containsKey(fieldName)) return; + Matching subFieldMatching = new Matching(); + subFieldMatching.merge(this.matching); + String subName = getName().concat(".").concat(fieldName); + var subField = new SDField(sdoc, subName, fieldType, null, + subFieldMatching, structFieldDepth + 1); + structFields.put(fieldName, subField); + }; + + if (dataType instanceof MapDataType) { + MapDataType mdt = (MapDataType) dataType; + supplyStructField.accept("key", mdt.getKeyType()); + supplyStructField.accept("value", mdt.getValueType()); + } else { + if (structFieldDepth >= 10) { + // too risky, infinite recursion + doneStructFields = true; + return; + } + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } + if ((dataType instanceof MapDataType) || (dataType instanceof CollectionDataType)) { + // "array of map" or "array of array" will not have any struct fields + // TODO: consider what this would mean + doneStructFields = true; + return; + } + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (dataType instanceof TemporaryUnknownType && subType != null) { + for (Field field : subType.fieldSet()) { + supplyStructField.accept(field.getName(), field.getDataType()); + } + } else if (dataType instanceof OwnedTemporaryType && subType != null) { + for (Field field : subType.fieldSet()) { + supplyStructField.accept(field.getName(), field.getDataType()); + } + } else if (dataType instanceof StructDataType) { + var sdt = (StructDataType) dataType; + for (Field field : sdt.getFields()) { + supplyStructField.accept(field.getName(), field.getDataType()); + } + } + if ((subType == null) && (structFields.size() > 0)) { + throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + + this + " [" + getDataType() + getDataType().getClass() + + "] with " + structFields.size() + " struct fields"); + } + // populate struct fields with matching + if (subType != null) { + for (Field f : subType.fieldSet()) { + if (f instanceof SDField) { + SDField field = (SDField) f; + SDField subField = structFields.get(field.getName()); + if (subField != null) { + // we just made this with a copy of our matching (see above) + Matching subFieldMatching = subField.getMatching(); + subFieldMatching.merge(field.getMatching()); + subField.setMatching(subFieldMatching); + } + } else { + throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); + } + } + } + // else ("missing subtype for struct fields in: " + this + " type " + getDataType() + " [" + getDataType().getClass().getSimpleName() + "]"); + } + doneStructFields = true; + } + + private Matching matchingForStructFields = null; + + public void addOperation(FieldOperation op) { + pendingOperations.add(op); + } + + @Override + public void applyOperations(SDField field) { + if (pendingOperations.isEmpty()) return; + + Collections.sort(pendingOperations); + ListIterator<FieldOperation> ops = pendingOperations.listIterator(); + while (ops.hasNext()) { + FieldOperation op = ops.next(); + ops.remove(); + op.apply(field); + } + } + + public void applyOperations() { + applyOperations(this); + } + + public void setId(int fieldId, DocumentType owner) { + super.setId(fieldId, owner); + idOverride = true; + } + + public StructDataType getFirstStructRecursive() { + DataType dataType = getDataType(); + while (true) { // Currently no nesting of collections + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } else if (dataType instanceof MapDataType) { + dataType = ((MapDataType)dataType).getValueType(); + } else { + break; + } + } + return (dataType instanceof StructDataType) ? (StructDataType)dataType : null; + } + + private DataType getFirstStructOrMapRecursive() { + DataType dataType = getDataType(); + while (dataType instanceof CollectionDataType) { // Currently no nesting of collections + dataType = ((CollectionDataType)dataType).getNestedType(); + } + return (dataType instanceof StructDataType || dataType instanceof MapDataType) ? dataType : null; + } + + private boolean usesStruct() { + DataType dt = getFirstStructRecursive(); + return (dt != null); + } + + @Override + public boolean usesStructOrMap() { + DataType dt = getFirstStructOrMapRecursive(); + return (dt != null); + } + + @Override + public boolean wasConfiguredToDoAttributing() { + return wasConfiguredToDoAttributing; + } + + /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */ + public void parseIndexingScript(String script) { + parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + } + + public void parseIndexingScript(String script, Linguistics linguistics, Map<String, Embedder> embedders) { + try { + ScriptParserContext config = new ScriptParserContext(linguistics, embedders); + config.setInputStream(new IndexingInput(script)); + setIndexingScript(ScriptExpression.newInstance(config)); + } catch (ParseException e) { + throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); + } + } + + /** Sets the indexing script of this, or null to not use a script */ + public void setIndexingScript(ScriptExpression exp) { + if (exp == null) { + exp = new ScriptExpression(); + } + indexingScript = exp; + if (indexingScript.isEmpty()) { + return; // TODO: This causes empty expressions not to be propagate to struct fields!! BAD BAD BAD!! + } + if (!wasConfiguredToDoAttributing()) { + wasConfiguredToDoAttributing = doesAttributing(); + } + if (!usesStructOrMap()) { + new ExpressionVisitor() { + + @Override + protected void doVisit(Expression exp) { + if (!(exp instanceof AttributeExpression)) { + return; + } + String fieldName = ((AttributeExpression)exp).getFieldName(); + if (fieldName == null) { + fieldName = getName(); + } + Attribute attribute = attributes.get(fieldName); + if (attribute == null) { + addAttribute(new Attribute(fieldName, getDataType())); + } + } + }.visit(indexingScript); + } + for (SDField structField : getStructFields()) { + structField.setIndexingScript(exp); + } + } + + @Override + public ScriptExpression getIndexingScript() { return indexingScript; } + + @SuppressWarnings("deprecation") + @Override + public void setDataType(DataType type) { + if (type.equals(DataType.URI)) { // Different defaults, naturally + normalizing.inferLowercase(); + stemming = Stemming.NONE; + } + this.dataType = type; + if ( ! idOverride) { + this.fieldId = calculateIdV7(null); + } + } + + @Override + public boolean isIndexStructureField() { + return indexStructureField; + } + + public void setIndexStructureField(boolean indexStructureField) { + this.indexStructureField = indexStructureField; + } + + @Override + public boolean hasIndex() { + return (getIndexingScript() != null) && doesIndexing(); + } + + /** Sets the literal boost of this field */ + public void setLiteralBoost(int literalBoost) { this.literalBoost=literalBoost; } + + /** + * Returns the literal boost of this field. This boost is added to a literal score + * when a query term matched as query term exactly (unnormalized and unstemmed). + * Default is non-positive. + */ + @Override + public int getLiteralBoost() { return literalBoost; } + + /** Sets the weight of this field */ + public void setWeight(int weight) { this.weight=weight; } + + /** Returns the weight of this field, or 0 if nothing is set */ + @Override + public int getWeight() { return weight; } + + /** + * Returns what kind of matching type should be applied. + */ + @Override + public Matching getMatching() { return matching; } + + /** + * Sets what kind of matching type should be applied. + * (Token matching is default, PREFIX, SUBSTRING, SUFFIX are alternatives) + */ + public void setMatching(Matching matching) { this.matching=matching; } + + /** + * Returns Dictionary settings. + */ + public Dictionary getDictionary() { return dictionary; } + public Dictionary getOrSetDictionary() { + if (dictionary == null) { + dictionary = new Dictionary(); + } + return dictionary; + } + + /** + * Set the matching type for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields at lookup time instead. + public void setMatchingType(MatchType type) { + this.getMatching().setType(type); + for (SDField structField : getStructFields()) { + structField.setMatchingType(type); + } + } + + /** + * Set the matching type for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields at lookup time instead. + public void setMatchingCase(Case casing) { + this.getMatching().setCase(casing); + for (SDField structField : getStructFields()) { + structField.setMatchingCase(casing); + } + } + /** + * Set matching algorithm for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields at lookup time instead. + public void setMatchingAlgorithm(MatchAlgorithm algorithm) { + this.getMatching().setAlgorithm(algorithm); + for (SDField structField : getStructFields()) { + structField.getMatching().setAlgorithm(algorithm); + } + } + + /** Adds an explicit index defined in this field */ + public void addIndex(Index index) { + indices.put(index.getName(),index); + } + + /** + * Returns an index, or null if no index with this name has had + * some <b>explicit settings</b> applied in this field (even if this returns null, + * the index may be implicitly defined by an indexing statement) + */ + @Override + public Index getIndex(String name) { + return indices.get(name); + } + + /** + * Returns an index if this field has one (implicitly or + * explicitly) targeting the given name. + */ + @Override + public boolean existsIndex(String name) { + if (indices.get(name) != null) return true; + return name.equals(getName()) && doesIndexing(); + } + + /** + * Defined indices on this field + * @return defined indices on this + */ + @Override + public Map<String, Index> getIndices() { + return indices; + } + + /** + * Sets the default rank type of this fields indices, and sets this rank type + * to all indices explicitly defined here which has no index set. + * (This complex behavior is dues to the fact than we would prefer to have rank types + * per field, not per index) + */ + public void setRankType(RankType rankType) { + this.rankType=rankType; + for (Index index : getIndices().values()) { + if (index.getRankType()==null) + index.setRankType(rankType); + } + + } + + /** Returns the rank settings set in a "rank" block for this field. This is never null. */ + @Override + public Ranking getRanking() { return ranking; } + + /** Returns the default rank type of indices of this field, or null if nothing is set */ + @Override + public RankType getRankType() { return this.rankType; } + + /** + * Returns the search-time attribute settings of this field or null if none is set. + * + * <p>TODO: Make unmodifiable.</p> + */ + @Override + public Map<String, Attribute> getAttributes() { return attributes; } + + public Attribute getAttribute() { + return attributes.get(getName()); + } + + public void addAttribute(Attribute attribute) { + String name = attribute.getName(); + if (name == null || "".equals(name)) { + name = getName(); + attribute.setName(name); + } + attributes.put(attribute.getName(),attribute); + } + + /** + * Returns the stemming setting of this field. + * Default is determined by the owning search definition. + * + * @return the stemming setting of this, or null, to use the default + */ + @Override + public Stemming getStemming() { return stemming; } + + /** + * Whether this field should be stemmed in this search definition + */ + @Override + public Stemming getStemming(Schema schema) { + if (stemming != null) + return stemming; + else + return schema.getStemming(); + } + + @Override + public Field asField() { + return this; + } + + /** + * Sets how this field should be stemmed, or set to null to use the default. + */ + public void setStemming(Stemming stemming) { + this.stemming = stemming; + } + + /** Returns an unmodifiable map of the summary fields defined in this */ + @Override + public Map<String, SummaryField> getSummaryFields() { + return Collections.unmodifiableMap(summaryFields); + } + + public void removeSummaryFields() { + summaryFields.clear(); + } + + /** Adds a summary field */ + public void addSummaryField(SummaryField summaryField) { + summaryFields.put(summaryField.getName(),summaryField); + } + + /** + * Returns a summary field defined (implicitly or explicitly) by this field. + * Returns null if there is no such summary field defined. + */ + @Override + public SummaryField getSummaryField(String name) { + return summaryFields.get(name); + } + + /** + * Returns a summary field defined (implicitly or explicitly) by this field. + * + * @param create true to create the summary field and add it to this field before returning if it is missing + * @return the summary field, or null if not present and create is false + */ + public SummaryField getSummaryField(String name, boolean create) { + SummaryField summaryField=summaryFields.get(name); + if (summaryField==null && create) { + summaryField=new SummaryField(name, getDataType()); + addSummaryField(summaryField); + } + return summaryFields.get(name); + } + + /** Returns list of static struct fields */ + @Override + public Collection<SDField> getStructFields() { + actuallyMakeStructFields(); + return structFields.values(); + } + + /** + * Returns a struct field defined in this field, + * potentially traversing into nested structs. + * Returns null if there is no such struct field defined. + */ + @Override + public SDField getStructField(String name) { + actuallyMakeStructFields(); + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + SDField superField = structFields.get(superFieldName); + if (superField != null) { + return superField.getStructField(subFieldName); + } + return null; + } + return structFields.get(name); + } + + /** + * Returns how the content of this field should be accent normalized etc + */ + @Override + public NormalizeLevel getNormalizing() { return normalizing; } + + /** + * Change how the content of this field should be accent normalized etc + */ + public void setNormalizing(NormalizeLevel level) { normalizing = level; } + + public void addQueryCommand(String name) { + queryCommands.add(name); + } + + public boolean hasQueryCommand(String name) { + return queryCommands.contains(name); + } + + /** Returns a list of query commands */ + @Override + public List<String> getQueryCommands() { return queryCommands; } + + @Override + public boolean equals(Object other) { + if ( ! (other instanceof SDField)) return false; + return super.equals(other); + } + + @Override + public int hashCode() { + return getName().hashCode(); + } + + @Override + public String toString() { + return "field '" + getName() + "'"; + } + + /** The aliases declared for this field */ + @Override + public Map<String, String> getAliasToName() { + return aliasToName; + } + + @Override + public boolean hasFullIndexingDocprocRights() { + Attribute self = getAttributes().get(getName()); + return (!isExtraField() || ((self != null) && self.isMutable())); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Sorting.java b/config-model/src/main/java/com/yahoo/schema/document/Sorting.java new file mode 100644 index 00000000000..2d0c9a5d27b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Sorting.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.io.Serializable; + +/** + * A search-time document attribute sort specification(per-document in-memory value). + * This belongs to the attribute or field(implicitt attribute). + * + * @author baldersheim + */ +public final class Sorting implements Cloneable, Serializable { + + // Remember to change hashCode and equals when you add new fields + public enum Function {UCA, RAW, LOWERCASE} + public enum Strength {PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL} + private boolean ascending = true; + private Function function = Function.UCA; + private String locale = ""; + private Strength strength = Strength.PRIMARY; + + public boolean isAscending() { return ascending; } + public boolean isDescending() { return ! ascending; } + public String getLocale() { return locale; } + public Function getFunction() { return function; } + public Strength getStrength() { return strength; } + + public void setAscending() { ascending = true; } + public void setDescending() { ascending = false; } + public void setFunction(Function function) { this.function = function; } + public void setLocale(String locale) { this.locale = locale; } + public void setStrength(Strength strength) { this.strength = strength; } + + public int hashCode() { + return locale.hashCode() + + strength.hashCode() + + function.hashCode() + + (isDescending() ? 13 : 0); + } + + public boolean equals(Object object) { + if (! (object instanceof Sorting)) return false; + + Sorting other=(Sorting)object; + return this.locale.equals(other.locale) && + (ascending == other.ascending) && + (function == other.function) && + (strength == other.strength); + } + + @Override + public Sorting clone() { + try { + return (Sorting)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + public String toString() { + return "sorting '" + (isAscending() ? '+' : '-') + function.toString() + "(" + strength.toString() + ", " + locale + ")"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Stemming.java b/config-model/src/main/java/com/yahoo/schema/document/Stemming.java new file mode 100644 index 00000000000..5ec844e2540 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Stemming.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.language.process.StemMode; + +/** + * The stemming setting of a field. This describes how the search engine + * should transform content of this field into base forms (stems) to increase + * recall (find "car" when you search for "cars" etc.). + * + * @author bratseth + */ +public enum Stemming { + + /** No stemming */ + NONE("none"), + + /** select shortest possible stem */ + SHORTEST("shortest"), + + /** select the "best" stem alternative */ + BEST("best"), + + /** index multiple stems */ + MULTIPLE("multiple"); + + private final String name; + + /** + * Returns the stemming object for the given string. + * The legal stemming names are the stemming constants in any capitalization. + * + * @throws IllegalArgumentException if there is no stemming type with the given name + */ + public static Stemming get(String stemmingName) { + try { + return Stemming.valueOf(stemmingName.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("'" + stemmingName + "' is not a valid stemming setting"); + } + } + + Stemming(String name) { + this.name = name; + } + + public String getName() { return name; } + + @Override + public String toString() { + return "stemming " + name; + } + + public StemMode toStemMode() { + switch(this) { + case SHORTEST: return StemMode.SHORTEST; + case MULTIPLE: return StemMode.ALL; + case BEST : return StemMode.BEST; + case NONE: return StemMode.NONE; + default: throw new IllegalStateException("Inconvertible stem mode " + this); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java new file mode 100644 index 00000000000..efc0674586d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * A field that is imported from a field in a referenced document type and given an alias name. + * + * This is temporary AST structure that only refers to the imported field by name. + * + * @author geirst + */ +public class TemporaryImportedField { + + private final String fieldName; + private final String referenceFieldName; + private final String targetFieldName; + + public TemporaryImportedField(String fieldName, + String referenceFieldName, + String targetFieldName) { + this.fieldName = fieldName; + this.referenceFieldName = referenceFieldName; + this.targetFieldName = targetFieldName; + } + + public String fieldName() { + return fieldName; + } + + public String referenceFieldName() { + return referenceFieldName; + } + + public String targetFieldName() { + return targetFieldName; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java new file mode 100644 index 00000000000..7ad4feb6d32 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.Schema; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * A set of fields that are imported from referenced document types. + * + * This is temporary AST structure that only refers to the imported fields by name. + * + * @author geirst + */ +public class TemporaryImportedFields { + + private final Schema owner; + private final Map<String, TemporaryImportedField> fields = new LinkedHashMap<>(); + + public TemporaryImportedFields(Schema owner) { + this.owner = owner; + } + + public void add(TemporaryImportedField importedField) { + fields.put(importedField.fieldName(), importedField); + } + + public boolean hasField(String fieldName) { + return fields.get(fieldName) != null; + } + + public Map<String, TemporaryImportedField> fields() { + if (owner.inherited().isEmpty()) return Collections.unmodifiableMap(fields); + if (owner.inherited().get().temporaryImportedFields().isEmpty()) return Collections.unmodifiableMap(fields); + + var allFields = new HashMap<>(owner.inherited().get().temporaryImportedFields().get().fields()); + allFields.putAll(fields); + return allFields; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java new file mode 100644 index 00000000000..3194a14a143 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataTypeName; + +/** + * @author baldersheim + */ +public class TemporarySDDocumentType extends SDDocumentType { + public TemporarySDDocumentType(DataTypeName name) { + super(name); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java new file mode 100644 index 00000000000..e455fa78455 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; + +/** + * @author Einar M R Rosenvinge + */ +public class TemporarySDField extends SDField { + + public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + super(repo, name, dataType, owner); + } + + public TemporarySDField(SDDocumentType repo, String name, DataType dataType) { + super(repo, name, dataType); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java new file mode 100644 index 00000000000..8de8c7b64fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; + +/** + * Common interface for various typed key (or field definitions). + * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping + * + * @author bratseth + */ +public interface TypedKey { + + String getName(); + + void setDataType(DataType type); + + DataType getDataType(); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java b/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java new file mode 100644 index 00000000000..3dc46a91c1b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document.annotation; + +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author Einar M R Rosenvinge + */ +public class SDAnnotationType extends AnnotationType { + + private SDDocumentType sdDocType; + private String inherits; + + public SDAnnotationType(String name) { + super(name); + } + + public SDAnnotationType(String name, SDDocumentType dataType, String inherits) { + super(name); + this.sdDocType = dataType; + this.inherits = inherits; + } + + public SDDocumentType getSdDocType() { + return sdDocType; + } + + public void setSdDocType(SDDocumentType value) { + assert(sdDocType == null); + sdDocType = value; + } + + public String getInherits() { + return inherits; + } + + public void inherit(String inherits) { + this.inherits = inherits; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java b/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java new file mode 100644 index 00000000000..de9bd977823 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document.annotation; + +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author Einar M R Rosenvinge + */ +public class TemporaryAnnotationReferenceDataType extends AnnotationReferenceDataType { + + private final String target; + + public TemporaryAnnotationReferenceDataType(String target) { + this.target = target; + } + + public String getTarget() { + return target; + } + + @Override + public void setAnnotationType(AnnotationType type) { + super.setName("annotationreference<" + type.getName() + ">"); + super.setAnnotationType(type); + } + +} |