summaryrefslogtreecommitdiffstats
path: root/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
diff options
context:
space:
mode:
Diffstat (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java')
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java45
1 files changed, 40 insertions, 5 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index 8e54d7c00d6..9cd7fb24e42 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType;
import com.yahoo.document.TensorDataType;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.HnswIndexParams;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.vespa.model.container.search.QueryProfiles;
@@ -25,10 +26,11 @@ public class TensorFieldProcessor extends Processor {
public void process(boolean validate, boolean documentsOnly) {
if ( ! validate) return;
- for (SDField field : search.allConcreteFields()) {
+ for (var field : search.allConcreteFields()) {
if ( field.getDataType() instanceof TensorDataType ) {
validateIndexingScripsForTensorField(field);
validateAttributeSettingForTensorField(field);
+ processIndexSettingsForTensorField(field);
}
else if (field.getDataType() instanceof CollectionDataType){
validateDataTypeForCollectionField(field);
@@ -37,20 +39,53 @@ public class TensorFieldProcessor extends Processor {
}
private void validateIndexingScripsForTensorField(SDField field) {
- if (field.doesIndexing()) {
- fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field.");
+ if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) {
+ fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " +
+ "Currently, only tensors with 1 indexed dimension supports that.");
}
}
+ private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) {
+ var type = ((TensorDataType)field.getDataType()).getTensorType();
+ // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search).
+ if ((type.dimensions().size() == 1) &&
+ type.dimensions().get(0).isIndexed()) {
+ return true;
+ }
+ return false;
+ }
+
+ private String tensorTypeToString(ImmutableSDField field) {
+ return ((TensorDataType)field.getDataType()).getTensorType().toString();
+ }
+
private void validateAttributeSettingForTensorField(SDField field) {
if (field.doesAttributing()) {
- Attribute attribute = field.getAttributes().get(field.getName());
+ var attribute = field.getAttributes().get(field.getName());
if (attribute != null && attribute.isFastSearch()) {
fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'.");
}
}
}
+ private void processIndexSettingsForTensorField(SDField field) {
+ if (!field.doesIndexing()) {
+ return;
+ }
+ if (isTensorTypeThatSupportsHnswIndex(field)) {
+ if (!field.doesAttributing()) {
+ fail(search, field, "A tensor that has an index must also be an attribute.");
+ }
+ var index = field.getIndex(field.getName());
+ // TODO: Calculate default params based on tensor dimension size
+ var params = new HnswIndexParams();
+ if (index != null && index.getHnswIndexParams().isPresent()) {
+ params = params.overrideFrom(index.getHnswIndexParams().get());
+ }
+ field.getAttribute().setHnswIndexParams(params);
+ }
+ }
+
private void validateDataTypeForCollectionField(SDField field) {
if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType)
fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");