diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java')
-rw-r--r-- | config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java | 324 |
1 files changed, 324 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java new file mode 100644 index 00000000000..99f73a75669 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -0,0 +1,324 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Ranking; +import com.yahoo.schema.document.Sorting; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isSupportedComplexField; + +/** + * The set of all attribute fields defined by a search definition + * + * @author bratseth + */ +public class AttributeFields extends Derived implements AttributesConfig.Producer { + + public enum FieldSet {ALL, FAST_ACCESS} + + private Map<String, Attribute> attributes = new java.util.LinkedHashMap<>(); + private Map<String, Attribute> importedAttributes = new java.util.LinkedHashMap<>(); + + /** Whether this has any position attribute */ + private boolean hasPosition = false; + + public static final AttributeFields empty = new AttributeFields(null); + + public AttributeFields(Schema schema) { + if (schema != null) + derive(schema); + } + + /** Derives everything from a field */ + @Override + protected void derive(ImmutableSDField field, Schema schema) { + if (unsupportedFieldType(field)) { + return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) + } + if (isArrayOfSimpleStruct(field)) { + deriveArrayOfSimpleStruct(field); + } else if (isMapOfSimpleStruct(field)) { + deriveMapOfSimpleStruct(field); + } else if (isMapOfPrimitiveType(field)) { + deriveMapOfPrimitiveType(field); + } else { + deriveAttributes(field); + } + } + + private static boolean unsupportedFieldType(ImmutableSDField field) { + return (field.usesStructOrMap() && + !isSupportedComplexField(field) && + !GeoPos.isAnyPos(field)); + } + + /** Returns an attribute by name, or null if it doesn't exist */ + public Attribute getAttribute(String attributeName) { + return attributes.get(attributeName); + } + + public boolean containsAttribute(String attributeName) { + return getAttribute(attributeName) != null; + } + + /** Derives one attribute. TODO: Support non-default named attributes */ + private void deriveAttributes(ImmutableSDField field) { + if (field.isImportedField()) { + deriveImportedAttributes(field); + return; + } + for (Attribute fieldAttribute : field.getAttributes().values()) { + deriveAttribute(field, fieldAttribute); + } + + if (field.containsExpression(ToPositionExpression.class)) { + // TODO: Move this check to processing and remove this + if (hasPosition) { + throw new IllegalArgumentException("Can not specify more than one set of position attributes per field: " + field.getName()); + } + hasPosition = true; + } + } + + private void applyRanking(ImmutableSDField field, Attribute attribute) { + Ranking ranking = field.getRanking(); + if (ranking != null && ranking.isFilter()) { + attribute.setEnableBitVectors(true); + attribute.setEnableOnlyBitVector(true); + } + } + + private void deriveAttribute(ImmutableSDField field, Attribute fieldAttribute) { + Attribute attribute = getAttribute(fieldAttribute.getName()); + if (attribute == null) { + attributes.put(fieldAttribute.getName(), fieldAttribute); + attribute = getAttribute(fieldAttribute.getName()); + } + applyRanking(field, attribute); + } + + private void deriveImportedAttributes(ImmutableSDField field) { + for (Attribute attribute : field.getAttributes().values()) { + if (!importedAttributes.containsKey(field.getName())) { + importedAttributes.put(field.getName(), attribute); + } + } + } + + private void deriveArrayOfSimpleStruct(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + deriveAttributeAsArrayType(structField); + } + } + + private void deriveAttributeAsArrayType(ImmutableSDField field) { + if (field.isImportedField()) { + deriveImportedAttributes(field); + return; + } + Attribute attribute = field.getAttributes().get(field.getName()); + if (attribute != null) { + applyRanking(field, attribute); + attributes.put(attribute.getName(), attribute.convertToArray()); + } + } + + private void deriveMapOfSimpleStruct(ImmutableSDField field) { + deriveAttributeAsArrayType(field.getStructField("key")); + deriveMapValueField(field.getStructField("value")); + } + + private void deriveMapValueField(ImmutableSDField valueField) { + for (ImmutableSDField structField : valueField.getStructFields()) { + deriveAttributeAsArrayType(structField); + } + } + + private void deriveMapOfPrimitiveType(ImmutableSDField field) { + deriveAttributeAsArrayType(field.getStructField("key")); + deriveAttributeAsArrayType(field.getStructField("value")); + } + + /** Returns a read only attribute iterator */ + public Iterator attributeIterator() { + return attributes().iterator(); + } + + public Collection<Attribute> attributes() { + return Collections.unmodifiableCollection(attributes.values()); + } + + public Collection<Attribute> structFieldAttributes(String baseFieldName) { + String structPrefix = baseFieldName + "."; + return attributes().stream() + .filter(attribute -> attribute.getName().startsWith(structPrefix)) + .collect(Collectors.toList()); + } + + public String toString() { + return "attributes " + getName(); + } + + @Override + protected String getDerivedName() { + return "attributes"; + } + + @SuppressWarnings("removal") // TODO Vespa 8: remove + private Map<String, AttributesConfig.Attribute.Builder> toMap(List<AttributesConfig.Attribute.Builder> ls) { + Map<String, AttributesConfig.Attribute.Builder> ret = new LinkedHashMap<>(); + for (AttributesConfig.Attribute.Builder builder : ls) { + ret.put((String) ConfigInstanceUtil.getField(builder, "name"), builder); + } + return ret; + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + //TODO This is just to get some exporting tests to work, Should be undone and removed + getConfig(builder, FieldSet.ALL, 77777, false); + } + + private boolean isAttributeInFieldSet(Attribute attribute, FieldSet fs) { + return (fs == FieldSet.ALL) || ((fs == FieldSet.FAST_ACCESS) && attribute.isFastAccess()); + } + + private AttributesConfig.Attribute.Builder getConfig(String attrName, Attribute attribute, boolean imported) { + AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder() + .name(attrName) + .datatype(AttributesConfig.Attribute.Datatype.Enum.valueOf(attribute.getType().getExportAttributeTypeName())) + .collectiontype(AttributesConfig.Attribute.Collectiontype.Enum.valueOf(attribute.getCollectionType().getName())); + if (attribute.isRemoveIfZero()) { + aaB.removeifzero(true); + } + if (attribute.isCreateIfNonExistent()) { + aaB.createifnonexistent(true); + } + aaB.enablebitvectors(attribute.isEnabledBitVectors()); + aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector()); + if (attribute.isFastSearch() || attribute.isFastRank()) { + // TODO make a separate fastrank flag in config instead of overloading fastsearch + aaB.fastsearch(true); + } + if (attribute.isFastAccess()) { + aaB.fastaccess(true); + } + if (attribute.isMutable()) { + aaB.ismutable(true); + } + if (attribute.isHuge()) { + aaB.huge(true); + } + if (attribute.isPaged()) { + aaB.paged(true); + } + if (attribute.getSorting().isDescending()) { + aaB.sortascending(false); + } + if (attribute.getSorting().getFunction() != Sorting.Function.UCA) { + aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString())); + } + if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) { + aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString())); + } + if (!attribute.getSorting().getLocale().isEmpty()) { + aaB.sortlocale(attribute.getSorting().getLocale()); + } + aaB.arity(attribute.arity()); + aaB.lowerbound(attribute.lowerBound()); + aaB.upperbound(attribute.upperBound()); + aaB.densepostinglistthreshold(attribute.densePostingListThreshold()); + if (attribute.tensorType().isPresent()) { + aaB.tensortype(attribute.tensorType().get().toString()); + } + aaB.imported(imported); + var dma = attribute.distanceMetric(); + aaB.distancemetric(AttributesConfig.Attribute.Distancemetric.Enum.valueOf(dma.toString())); + if (attribute.hnswIndexParams().isPresent()) { + var ib = new AttributesConfig.Attribute.Index.Builder(); + var params = attribute.hnswIndexParams().get(); + ib.hnsw.enabled(true); + ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); + ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); + ib.hnsw.multithreadedindexing(params.multiThreadedIndexing()); + aaB.index(ib); + } + Dictionary dictionary = attribute.getDictionary(); + if (dictionary != null) { + aaB.dictionary.type(convert(dictionary.getType())); + aaB.dictionary.match(convert(dictionary.getMatch())); + } + aaB.match(convertMatch(attribute.getCase())); + return aaB; + } + + private static AttributesConfig.Attribute.Dictionary.Type.Enum convert(Dictionary.Type type) { + switch (type) { + case BTREE: + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + case HASH: + return AttributesConfig.Attribute.Dictionary.Type.HASH; + case BTREE_AND_HASH: + return AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH; + } + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + } + private static AttributesConfig.Attribute.Dictionary.Match.Enum convert(Case type) { + switch (type) { + case CASED: + return AttributesConfig.Attribute.Dictionary.Match.CASED; + case UNCASED: + return AttributesConfig.Attribute.Dictionary.Match.UNCASED; + } + return AttributesConfig.Attribute.Dictionary.Match.UNCASED; + } + private static AttributesConfig.Attribute.Match.Enum convertMatch(Case type) { + switch (type) { + case CASED: + return AttributesConfig.Attribute.Match.CASED; + case UNCASED: + return AttributesConfig.Attribute.Match.UNCASED; + } + return AttributesConfig.Attribute.Match.UNCASED; + } + + public void getConfig(AttributesConfig.Builder builder, FieldSet fs, long maxUnCommittedMemory, boolean enableBitVectors) { + for (Attribute attribute : attributes.values()) { + if (isAttributeInFieldSet(attribute, fs)) { + AttributesConfig.Attribute.Builder attrBuilder = getConfig(attribute.getName(), attribute, false); + attrBuilder.maxuncommittedmemory(maxUnCommittedMemory); + if (enableBitVectors && attribute.isFastSearch()) { + attrBuilder.enablebitvectors(true); + } + builder.attribute(attrBuilder); + } + } + if (fs == FieldSet.ALL) { + for (Map.Entry<String, Attribute> entry : importedAttributes.entrySet()) { + AttributesConfig.Attribute.Builder attrBuilder = getConfig(entry.getKey(), entry.getValue(), true); + attrBuilder.maxuncommittedmemory(maxUnCommittedMemory); + builder.attribute(attrBuilder); + } + } + } + +} |