diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java')
-rw-r--r-- | config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java | 45 |
1 files changed, 40 insertions, 5 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 8e54d7c00d6..9cd7fb24e42 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.TensorDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -25,10 +26,11 @@ public class TensorFieldProcessor extends Processor { public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; - for (SDField field : search.allConcreteFields()) { + for (var field : search.allConcreteFields()) { if ( field.getDataType() instanceof TensorDataType ) { validateIndexingScripsForTensorField(field); validateAttributeSettingForTensorField(field); + processIndexSettingsForTensorField(field); } else if (field.getDataType() instanceof CollectionDataType){ validateDataTypeForCollectionField(field); @@ -37,20 +39,53 @@ public class TensorFieldProcessor extends Processor { } private void validateIndexingScripsForTensorField(SDField field) { - if (field.doesIndexing()) { - fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field."); + if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { + fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that."); } } + private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { + var type = ((TensorDataType)field.getDataType()).getTensorType(); + // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). + if ((type.dimensions().size() == 1) && + type.dimensions().get(0).isIndexed()) { + return true; + } + return false; + } + + private String tensorTypeToString(ImmutableSDField field) { + return ((TensorDataType)field.getDataType()).getTensorType().toString(); + } + private void validateAttributeSettingForTensorField(SDField field) { if (field.doesAttributing()) { - Attribute attribute = field.getAttributes().get(field.getName()); + var attribute = field.getAttributes().get(field.getName()); if (attribute != null && attribute.isFastSearch()) { fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'."); } } } + private void processIndexSettingsForTensorField(SDField field) { + if (!field.doesIndexing()) { + return; + } + if (isTensorTypeThatSupportsHnswIndex(field)) { + if (!field.doesAttributing()) { + fail(search, field, "A tensor that has an index must also be an attribute."); + } + var index = field.getIndex(field.getName()); + // TODO: Calculate default params based on tensor dimension size + var params = new HnswIndexParams(); + if (index != null && index.getHnswIndexParams().isPresent()) { + params = params.overrideFrom(index.getHnswIndexParams().get()); + } + field.getAttribute().setHnswIndexParams(params); + } + } + private void validateDataTypeForCollectionField(SDField field) { if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); |