diff options
author | Geir Storli <geirst@yahooinc.com> | 2023-01-13 14:49:43 +0000 |
---|---|---|
committer | Geir Storli <geirst@yahooinc.com> | 2023-01-13 14:49:43 +0000 |
commit | 64a3304435dc3823ac0ea4958ea83fc57fae704c (patch) | |
tree | 6288edcacedf40e42626cf51920c1afe643feaea /config-model | |
parent | dbb6406528fc5723cf7310d6d53961e96c8ce257 (diff) |
Allow hnsw index for tensors with 1 mapped and 1 indexed dimension.
Diffstat (limited to 'config-model')
-rw-r--r-- | config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java | 13 | ||||
-rw-r--r-- | config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java | 18 |
2 files changed, 25 insertions, 6 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java index e0ce9917179..37da07f8227 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java @@ -44,17 +44,26 @@ public class TensorFieldProcessor extends Processor { private void validateIndexingScripsForTensorField(SDField field) { if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { fail(schema, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + - "Currently, only tensors with 1 indexed dimension supports that."); + "Currently, only tensors with 1 indexed dimension or 1 mapped + 1 indexed dimension support that."); } } private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { var type = ((TensorDataType)field.getDataType()).getTensorType(); - // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). + // Tensors with 1 indexed dimension support hnsw index (used for approximate nearest neighbor search). if ((type.dimensions().size() == 1) && type.dimensions().get(0).isIndexed()) { return true; } + // Tensors with 1 mapped + 1 indexed dimension support hnsw index (aka multiple vectors per document). + if (type.dimensions().size() == 2) { + var a = type.dimensions().get(0); + var b = type.dimensions().get(1); + if ((a.isMapped() && b.isIndexed()) || + (a.isIndexed() && b.isMapped())) { + return true; + } + } return false; } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java index 60e1e35fb2e..b7817900dac 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java @@ -35,7 +35,7 @@ public class TensorFieldTestCase { } catch (IllegalArgumentException e) { assertEquals("For schema 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + - "Currently, only tensors with 1 indexed dimension supports that.", + "Currently, only tensors with 1 indexed dimension or 1 mapped + 1 indexed dimension support that.", e.getMessage()); } } @@ -86,6 +86,12 @@ public class TensorFieldTestCase { } @Test + void tensor_with_one_mapped_and_one_indexed_dimension_can_have_hnsw_index() throws ParseException { + assertHnswIndexParams("tensor(x{},y[64])", "", 16, 200); + assertHnswIndexParams("tensor(x[64],y{})", "", 16, 200); + } + + @Test void hnsw_index_parameters_can_be_specified() throws ParseException { assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); @@ -157,7 +163,11 @@ public class TensorFieldTestCase { } private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { - var sd = getSdWithIndexSpec(indexSpec); + assertHnswIndexParams("tensor(x[64])", indexSpec, maxLinksPerNode, neighborsToExploreAtInsert); + } + + private void assertHnswIndexParams(String tensorType, String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { + var sd = getSdWithIndexSpec(tensorType, indexSpec); var search = createFromString(sd).getSchema(); var attr = search.getAttribute("t1"); var params = attr.hnswIndexParams(); @@ -166,8 +176,8 @@ public class TensorFieldTestCase { assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); } - private String getSdWithIndexSpec(String indexSpec) { - return getSd(joinLines("field t1 type tensor(x[64]) {", + private String getSdWithIndexSpec(String tensorType, String indexSpec) { + return getSd(joinLines("field t1 type " + tensorType + " {", " indexing: attribute | index", " " + indexSpec, "}")); |