Merge pull request #27382 from vespa-engine/bratseth/validate-prefix-matching-take-2-alternative-ending

Bratseth/validate prefix matching take 2 alternative ending

11 files changed, 160 insertions, 73 deletions

diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
index 06ae9923dae..c164fd3eb1c 100644
--- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
@@ -277,7 +277,7 @@ public class ClusterSearcherTestCase {
                     schemaBuilder.add(new RankProfile.Builder("testprofile").build());
                 schemas.add(schemaBuilder.build());
             }
-            return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, Map.of())));
+            return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, List.of())));
         } finally {
             cluster.deconstruct();
         }
@@ -462,7 +462,7 @@ public class ClusterSearcherTestCase {
                                    qrSearchersConfig.build(),
                                    clusterConfig.build(),
                                    documentDbConfig.build(),
-                                   new SchemaInfo(List.of(schema.build()), Map.of()),
+                                   new SchemaInfo(List.of(schema.build()), List.of()),
                                    dispatchers,
                                    null,
                                    vipStatus,
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
index 73975ecaa96..7a63eb07641 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
@@ -129,7 +129,7 @@ public class FastSearcherTestCase {
                 new SummaryParameters(null),
                 new ClusterParams("testhittype"),
                 documentDb,
-                new SchemaInfo(List.of(schema.build()), Map.of()));
+                new SchemaInfo(List.of(schema.build()), List.of()));
         Query q = new Query("?query=foo");
         Result result = doSearch(backend, q, 0, 10);
         assertFalse(backend.summaryNeedsQuery(q));
@@ -210,7 +210,7 @@ public class FastSearcherTestCase {
     private SchemaInfo schemaInfo(String schemaName) {
         var schema = new Schema.Builder(schemaName);
         schema.add(new RankProfile.Builder("default").build());
-        return new SchemaInfo(List.of(schema.build()),  Map.of());
+        return new SchemaInfo(List.of(schema.build()),  List.of());
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
index d70b42aa36b..07394676e09 100644
--- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
@@ -261,14 +261,14 @@ public class QueryTestCase {
     @Test
     void testNoneHitsNegativeOffsetValue() {
         assertQueryError(
-                "?query=test&hits=(none)&offset=-10",
+                "?query=test&hits=(none)",
                 "Could not set 'hits' to '(none)': '(none)' is not a valid integer");
     }
 
     @Test
     void testFeedbackIsTransferredToResult() {
         assertQueryError(
-                "?query=test&hits=(none)&offset=-10",
+                "?query=test&hits=(none)",
                 "Could not set 'hits' to '(none)': '(none)' is not a valid integer");
     }
 
diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java
index cbe4ddcbc63..03b53970550 100644
--- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java
+++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java
@@ -5,6 +5,7 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.language.Language;
 import com.yahoo.language.process.Embedder;
 import com.yahoo.search.Query;
+import com.yahoo.search.schema.Cluster;
 import com.yahoo.search.schema.RankProfile;
 import com.yahoo.search.schema.Schema;
 import com.yahoo.search.schema.SchemaInfo;
@@ -259,9 +260,9 @@ public class RankProfileInputTest {
                                          .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})"))
                                          .build())
                             .build());
-        Map<String, List<String>> clusters = new HashMap<>();
-        clusters.put("ab", List.of("a", "b"));
-        clusters.put("a", List.of("a"));
+        List<Cluster> clusters = new ArrayList<>();
+        clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build());
+        clusters.add(new Cluster.Builder("a").addSchema("a").build());
         return new SchemaInfo(schemas, clusters);
     }
 
diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java
index a46f3480d50..4aced1b5e25 100644
--- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java
+++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java
@@ -83,17 +83,14 @@ public class SchemaInfoTester {
                                          .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})"))
                                          .build())
                             .build());
-        Map<String, List<String>> clusters = new HashMap<>();
-        clusters.put("ab", List.of("a", "b"));
-        clusters.put("a", List.of("a"));
+        List<Cluster> clusters = new ArrayList<>();
+        clusters.add(new Cluster.Builder("ab").addSchema("a").addSchema("b").build());
+        clusters.add(new Cluster.Builder("a").addSchema("a").setStreaming(true).build());
         return new SchemaInfo(schemas, clusters);
     }
 
     /** Creates the same schema info as createSchemaInfo from config objects. */
     static SchemaInfo createSchemaInfoFromConfig() {
-
-        var indexInfoConfig = new IndexInfoConfig.Builder();
-
         var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder();
         rankProfileCommon.name("commonProfile");
         rankProfileCommon.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})"));
@@ -141,7 +138,7 @@ public class SchemaInfoTester {
 
         schemaInfoInfoConfig.schema(schemaB);
 
-        // ----- Info about which schemas are in which clusters
+        // ----- Info about clusters
         var qrSearchersConfig = new QrSearchersConfig.Builder();
         var clusterAB = new QrSearchersConfig.Searchcluster.Builder();
         clusterAB.name("ab");
@@ -149,10 +146,11 @@ public class SchemaInfoTester {
         qrSearchersConfig.searchcluster(clusterAB);
         var clusterA = new QrSearchersConfig.Searchcluster.Builder();
         clusterA.name("a");
+        clusterA.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.STREAMING);
         clusterA.searchdef("a");
         qrSearchersConfig.searchcluster(clusterA);
 
-        return new SchemaInfo(indexInfoConfig.build(), schemaInfoInfoConfig.build(), qrSearchersConfig.build());
+        return new SchemaInfo(schemaInfoInfoConfig.build(), qrSearchersConfig.build());
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java
new file mode 100644
index 00000000000..9367d9f335a
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorFieldTypeTest.java
@@ -0,0 +1,57 @@
+package com.yahoo.search.searchers.test;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.schema.Field;
+import com.yahoo.search.schema.FieldSet;
+import com.yahoo.search.schema.Schema;
+import com.yahoo.search.schema.SchemaInfo;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchers.QueryValidator;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * @author bratseth
+ */
+public class QueryValidatorFieldTypeTest {
+
+    @Test
+    void testTensorsCannotBeSearchedForTerms() {
+        var test = new Schema.Builder("test")
+                           .add(new Field.Builder("mytensor1", "tensor(x[100])").build())
+                           .add(new Field.Builder("mytensor2", "tensor<float>(x[100])").build())
+                           .add(new Field.Builder("mystring", "string").addAlias("fieldAlias").build())
+                           .add(new FieldSet.Builder("myFieldSet").addField("mystring").build())
+                           .build();
+        var schemaInfo = new SchemaInfo(List.of(test), List.of());
+        Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo));
+
+        assertSucceeds("?query=mystring:foo", execution);
+        assertSucceeds("?query=fieldAlias:foo", execution);
+        assertSucceeds("?query=myFieldSet:foo", execution);
+        assertSucceeds("?query=none:foo", execution);
+        assertFails("Cannot search for terms in 'mytensor1': It is a tensor field",
+                    "?query=mytensor1:foo", execution);
+        assertFails("Cannot search for terms in 'mytensor2': It is a tensor field",
+                    "?query=mytensor2:foo", execution);
+    }
+    private void assertSucceeds(String query, Execution execution) {
+        new QueryValidator().search(new Query(query), execution);
+    }
+
+    private void assertFails(String expectedError, String query, Execution execution) {
+        try {
+            new QueryValidator().search(new Query(query), execution);
+            fail("Expected validation error from " + query);
+        }
+        catch (IllegalArgumentException e) {
+            // success
+            assertEquals(expectedError, e.getMessage());
+        }
+    }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java
new file mode 100644
index 00000000000..b653e4d97aa
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorPrefixTest.java
@@ -0,0 +1,83 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers.test;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.schema.Cluster;
+import com.yahoo.search.schema.Field;
+import com.yahoo.search.schema.Schema;
+import com.yahoo.search.schema.SchemaInfo;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchers.QueryValidator;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Disabled;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * @author bratseth
+ */
+public class QueryValidatorPrefixTest {
+
+    @Disabled
+    @Test
+    void testPrefixRequiresAttribute() {
+        var indexing = new Cluster.Builder("indexing").addSchema("test1").build();
+        var streaming = new Cluster.Builder("streaming").addSchema("test1").addSchema("test2").setStreaming(true).build();
+        var schemaInfo = new SchemaInfo(List.of(schema("test1"), schema("test2")), List.of(indexing, streaming));
+
+        assertIndexingValidation("", schemaInfo);
+        assertIndexingValidation("sources=indexing", schemaInfo);
+        assertIndexingValidation("sources=indexing,streaming", schemaInfo);
+        assertIndexingValidation("sources=indexing,streaming,ignored", schemaInfo);
+        assertStreamingValidation("sources=streaming", schemaInfo);
+        assertStreamingValidation("sources=streaming,ignored", schemaInfo);
+        assertIndexingValidation("sources=test1", schemaInfo);
+        assertIndexingValidation("sources=test1,streaming", schemaInfo);
+        assertStreamingValidation("sources=test2,streaming", schemaInfo);
+        assertIndexingValidation("sources=test1,test2", schemaInfo);
+        assertStreamingValidation("sources=test2", schemaInfo);
+    }
+
+    private Schema schema(String name) {
+        return new Schema.Builder(name)
+                       .add(new Field.Builder("attributeOnly", "string").setAttribute(true).build())
+                       .add(new Field.Builder("indexOnly", "string").setIndex(true).build())
+                       .add(new Field.Builder("attributeAndIndex", "string").setAttribute(true).setIndex(true).build())
+                       .build();
+    }
+
+    private void assertIndexingValidation(String sourcesParameter, SchemaInfo schemaInfo) {
+        Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo));
+        assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution);
+        assertFails("'indexOnly' is not an attribute field: Prefix matching is not supported",
+                    "?query=indexOnly:foo*&" + sourcesParameter, execution);
+        assertFails("'attributeAndIndex' is an index field: Prefix matching is not supported even when it is also an attribute",
+                    "?query=attributeAndIndex:foo*&" + sourcesParameter, execution);
+    }
+
+    private void assertStreamingValidation(String sourcesParameter, SchemaInfo schemaInfo) {
+        Execution execution = new Execution(Execution.Context.createContextStub(schemaInfo));
+        assertSucceeds("?query=attributeOnly:foo*&" + sourcesParameter, execution);
+        assertSucceeds("?query=indexOnly:foo*&" + sourcesParameter, execution);
+        assertSucceeds("?query=attributeAndIndex:foo*&" + sourcesParameter, execution);
+    }
+
+    private void assertSucceeds(String query, Execution execution) {
+        new QueryValidator().search(new Query(query), execution);
+    }
+
+    private void assertFails(String expectedError, String query, Execution execution) {
+        try {
+            new QueryValidator().search(new Query(query), execution);
+            fail("Expected validation error from " + query);
+        }
+        catch (IllegalArgumentException e) {
+            // success
+            assertEquals(expectedError, e.getMessage());
+        }
+    }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java
deleted file mode 100644
index 64fb4354003..00000000000
--- a/container-search/src/test/java/com/yahoo/search/searchers/test/QueryValidatorTestCase.java
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.searchers.test;
-
-import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.IndexModel;
-import com.yahoo.prelude.SearchDefinition;
-import com.yahoo.search.Query;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.search.searchers.QueryValidator;
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.fail;
-
-/**
- * @author bratseth
- */
-public class QueryValidatorTestCase {
-
-    @Test
-    void testValidation() {
-        SearchDefinition sd = new SearchDefinition("test");
-        sd.addCommand("mytensor1", "type tensor(x[100]");
-        sd.addCommand("mytensor2", "type tensor<float>(x[100]");
-        sd.addCommand("mystring", "type string");
-        IndexModel model = new IndexModel(sd);
-
-        IndexFacts indexFacts = new IndexFacts(model);
-        Execution execution = new Execution(Execution.Context.createContextStub(indexFacts));
-        new QueryValidator().search(new Query("?query=mystring:foo"), execution);
-
-        try {
-            new QueryValidator().search(new Query("?query=mytensor1:foo"), execution);
-            fail("Expected validation error");
-        }
-        catch (IllegalArgumentException e) {
-            // success
-            assertEquals("Cannot search 'mytensor1': It is a tensor field", e.getMessage());
-        }
-
-        try {
-            new QueryValidator().search(new Query("?query=mytensor2:foo"), execution);
-            fail("Expected validation error");
-        }
-        catch (IllegalArgumentException e) {
-            // success
-            assertEquals("Cannot search 'mytensor2': It is a tensor field", e.getMessage());
-        }
-    }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java
index 0bb3095fa9d..3a7641e7dc0 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java
@@ -74,7 +74,7 @@ public class YqlFieldAndSourceTestCase {
                                                                      .addField(FIELD2, "string").build())
               .add((new DocumentSummary.Builder(SORTABLE_ATTRIBUTES_SUMMARY_CLASS).addField(FIELD2, "string").build()))
               .add((new DocumentSummary.Builder(THIRD_OPTION).addField(FIELD3, "string").build()));
-        return new SchemaInfo(List.of(schema.build()), Map.of());
+        return new SchemaInfo(List.of(schema.build()), List.of());
     }
 
     @AfterEach
diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
index 10d3a5aeabe..2eb136056ac 100644
--- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
+++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
@@ -76,9 +76,8 @@ public class SelectTestCase {
                 "my.nested.title:madonna");
     }
 
-
     @Test
-    void testOr() throws Exception {
+    void testOr() {
         ObjectNode json_two_or = jsonMapper.createObjectNode();
         ObjectNode json_three_or = jsonMapper.createObjectNode();
         ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna");
@@ -100,7 +99,7 @@ public class SelectTestCase {
     }
 
     @Test
-    void testAnd() throws Exception {
+    void testAnd() {
         ObjectNode json_two_and = jsonMapper.createObjectNode();
         ObjectNode json_three_and = jsonMapper.createObjectNode();
         ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna");
diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java
index 578ccec7f40..19c03faae66 100644
--- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java
@@ -240,7 +240,7 @@ public class VdsStreamingSearcherTestCase {
                 new SummaryParameters("default"),
                 new ClusterParams("clusterName"),
                 new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("test")).build(),
-                new SchemaInfo(List.of(schema.build()), Map.of()));
+                new SchemaInfo(List.of(schema.build()), List.of()));
 
         // Magic query values are used to trigger specific behaviors from mock visitor.
         checkError(searcher, "/?query=noselection",