diff options
Diffstat (limited to 'container-search/src/test/java/com')
8 files changed, 264 insertions, 76 deletions
diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java index d9046075f38..e5bbb48e807 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java @@ -13,8 +13,8 @@ import java.util.Iterator; import java.util.SortedSet; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.fail; /** * @author Tony Vaagenes @@ -59,37 +59,33 @@ public class SearchChainResolverTestCase { @Test void require_error_message_for_invalid_source() { - try { - resolve("no-such-source"); - fail("Expected exception."); - } catch (UnresolvedSearchChainException e) { - assertEquals("Could not resolve source ref 'no-such-source'.", e.getMessage()); - } + var result = resolve("no-such-source"); + assertEquals("Could not resolve source ref 'no-such-source'.", result.errorMsg()); } @Test - void lookup_search_chain() throws Exception { - SearchChainInvocationSpec res = resolve(searchChainId.getName()); + void lookup_search_chain() { + SearchChainInvocationSpec res = resolve(searchChainId.getName()).invocationSpec(); assertEquals(searchChainId, res.searchChainId); } //TODO: TVT: @Test() - public void lookup_provider() throws Exception { - SearchChainInvocationSpec res = resolve(providerId.getName()); + public void lookup_provider() { + SearchChainInvocationSpec res = resolve(providerId.getName()).invocationSpec(); assertEquals(providerId, res.provider); assertNull(res.source); assertEquals(providerId, res.searchChainId); } @Test - void lookup_source() throws Exception { - SearchChainInvocationSpec res = resolve(sourceId.getName()); + void lookup_source() { + SearchChainInvocationSpec res = resolve(sourceId.getName()).invocationSpec(); assertIsSourceInProvider(res); } @Test - void lookup_source_search_chain_directly() throws Exception { - SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()); + void lookup_source_search_chain_directly() { + SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()).invocationSpec(); assertIsSourceInProvider(res); } @@ -100,8 +96,8 @@ public class SearchChainResolverTestCase { } @Test - void lookup_source_for_provider2() throws Exception { - SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()); + void lookup_source_for_provider2() { + SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()).invocationSpec(); assertEquals(provider2Id, res.provider); assertEquals(sourceId, res.source); assertEquals(sourceChainInProvider2Id, res.searchChainId); @@ -126,22 +122,24 @@ public class SearchChainResolverTestCase { return new PropertyMap(); } - private SearchChainInvocationSpec resolve(String sourceSpecification) throws UnresolvedSearchChainException { + private ResolveResult resolve(String sourceSpecification) { return resolve(sourceSpecification, emptySourceToProviderMap()); } - private SearchChainInvocationSpec resolve(String sourceSpecification, String providerSpecification) - throws UnresolvedSearchChainException { + private ResolveResult resolve(String sourceSpecification, String providerSpecification) { Properties sourceToProviderMap = emptySourceToProviderMap(); sourceToProviderMap.set("source." + sourceSpecification + ".provider", providerSpecification); return resolve(sourceSpecification, sourceToProviderMap); } - private SearchChainInvocationSpec resolve(String sourceSpecification, Properties sourceToProviderMap) - throws UnresolvedSearchChainException { - SearchChainInvocationSpec res = searchChainResolver.resolve( + private ResolveResult resolve(String sourceSpecification, Properties sourceToProviderMap) { + ResolveResult res = searchChainResolver.resolve( ComponentSpecification.fromString(sourceSpecification), sourceToProviderMap); - assertEquals(federationOptions, res.federationOptions); + if (res.invocationSpec() != null) { + assertEquals(federationOptions, res.invocationSpec().federationOptions); + } else { + assertNotNull(res.errorMsg()); + } return res; } diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java index b32135afc94..95262937c01 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java @@ -3,21 +3,16 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.IndexModel; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; -import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeMap; import static com.yahoo.search.federation.sourceref.SearchChainResolverTestCase.emptySourceToProviderMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; /** * Test for SourceRefResolver. @@ -47,49 +42,38 @@ public class SourceRefResolverTestCase { @Test void lookup_search_chain() throws Exception { - Set<SearchChainInvocationSpec> searchChains = resolve(cluster1); + List<ResolveResult> searchChains = resolve(cluster1); assertEquals(1, searchChains.size()); assertTrue(searchChainIds(searchChains).contains(cluster1)); } @Test void lookup_search_chains_for_document1() throws Exception { - Set<SearchChainInvocationSpec> searchChains = resolve("document1"); + List<ResolveResult> searchChains = resolve("document1"); assertEquals(2, searchChains.size()); assertTrue(searchChainIds(searchChains).containsAll(List.of(cluster1, cluster2))); } @Test void error_when_document_gives_cluster_without_matching_search_chain() { - try { - resolve("document3"); - fail("Expected exception"); - } catch (UnresolvedSearchChainException e) { - assertEquals("Failed to resolve cluster search chain 'cluster3' " + - "when using source ref 'document3' as a document name.", - e.getMessage()); - } + List<ResolveResult> result = resolve("document3"); + + assertEquals("Failed to resolve cluster search chain 'cluster3' " + + "when using source ref 'document3' as a document name.", + result.get(0).errorMsg()); } @Test void error_when_no_document_or_search_chain() { - try { - resolve("document4"); - fail("Expected exception"); - } catch (UnresolvedSearchChainException e) { - assertEquals("Could not resolve source ref 'document4'.", e.getMessage()); - } + List<ResolveResult> results = resolve("document4"); + assertEquals("Could not resolve source ref 'document4'.", results.get(0).errorMsg()); } - private List<String> searchChainIds(Set<SearchChainInvocationSpec> searchChains) { - List<String> names = new ArrayList<>(); - for (SearchChainInvocationSpec searchChain : searchChains) { - names.add(searchChain.searchChainId.stringValue()); - } - return names; + private List<String> searchChainIds(Collection<ResolveResult> searchChains) { + return searchChains.stream().map(r -> r.invocationSpec().searchChainId.stringValue()).toList(); } - private Set<SearchChainInvocationSpec> resolve(String documentName) throws UnresolvedSearchChainException { + private List<ResolveResult> resolve(String documentName) { return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap()); } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java index c4b8c9f2044..027152bfd69 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java @@ -55,14 +55,13 @@ public class ValidateFuzzySearcherTestCase { searcher = new ValidateFuzzySearcher(); } - private String makeQuery(String attribute, String query, int maxEditDistance, int prefixLength) { - return "select * from sources * where " + attribute + - " contains ({maxEditDistance:" + maxEditDistance + ", prefixLength:" + prefixLength +"}" + - "fuzzy(\"" + query + "\"))"; + private String makeQuery(String attribute, String query, int maxEditDistance, int prefixLength, boolean prefixMatch) { + return "select * from sources * where %s contains ({maxEditDistance:%d,prefixLength:%d,prefix:%b}fuzzy(\"%s\"))" + .formatted(attribute, maxEditDistance, prefixLength, prefixMatch, query); } private String makeQuery(String attribute, String query) { - return makeQuery(attribute, query, 2, 0); + return makeQuery(attribute, query, 2, 0, false); } @@ -76,7 +75,7 @@ public class ValidateFuzzySearcherTestCase { if (validAttributes.contains(attribute)) { assertNull(r.hits().getError()); } else { - assertErrMsg("FUZZY(fuzzy,2,0) " + attribute + ":fuzzy field is not a string attribute", r); + assertErrMsg("FUZZY(fuzzy,2,0,false) " + attribute + ":fuzzy field is not a string attribute", r); } } } @@ -85,28 +84,28 @@ public class ValidateFuzzySearcherTestCase { void testInvalidEmptyStringQuery() { String q = makeQuery("string_single", ""); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(,2,0) string_single: fuzzy query must be non-empty", r); + assertErrMsg("FUZZY(,2,0,false) string_single: fuzzy query must be non-empty", r); } @Test void testInvalidQueryWrongMaxEditDistance() { - String q = makeQuery("string_single", "fuzzy", -1, 0); + String q = makeQuery("string_single", "fuzzy", -1, 0, false); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(fuzzy,-1,0) string_single:fuzzy has invalid maxEditDistance -1: Must be >= 0", r); + assertErrMsg("FUZZY(fuzzy,-1,0,false) string_single:fuzzy has invalid maxEditDistance -1: Must be >= 0", r); } @Test void testInvalidQueryWrongPrefixLength() { - String q = makeQuery("string_single", "fuzzy", 2, -1); + String q = makeQuery("string_single", "fuzzy", 2, -1, true); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(fuzzy,2,-1) string_single:fuzzy has invalid prefixLength -1: Must be >= 0", r); + assertErrMsg("FUZZY(fuzzy,2,-1,true) string_single:fuzzy has invalid prefixLength -1: Must be >= 0", r); } @Test void testInvalidQueryWrongAttributeName() { String q = makeQuery("wrong_name", "fuzzy"); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(fuzzy,2,0) wrong_name:fuzzy field is not a string attribute", r); + assertErrMsg("FUZZY(fuzzy,2,0,false) wrong_name:fuzzy field is not a string attribute", r); } private static void assertErrMsg(String message, Result r) { diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/en.json b/container-search/src/test/java/com/yahoo/search/significance/model/en.json new file mode 100644 index 00000000000..50bae5e3451 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/significance/model/en.json @@ -0,0 +1,14 @@ +{ + "version" : "1.0", + "id" : "test::1", + "description" : "desc", + "corpus-size" : 10, + "language" : "en", + "word-count" : 4, + "frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 2 + } +} diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java new file mode 100644 index 00000000000..890db3abb51 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -0,0 +1,153 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.significance.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Language; +import com.yahoo.language.significance.SignificanceModel; +import com.yahoo.language.significance.SignificanceModelRegistry; +import com.yahoo.language.significance.impl.DefaultSignificanceModelRegistry; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.significance.SignificanceSearcher; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.HashMap; + + +import static com.yahoo.test.JunitCompat.assertEquals; + +/** + * Tests significance term in the search chain. + * + * @author MariusArhaug + */ +public class SignificanceSearcherTest { + SignificanceModelRegistry significanceModelRegistry; + SignificanceSearcher searcher; + + public SignificanceSearcherTest() { + HashMap<Language, Path> map = new HashMap<>(); + map.put(Language.ENGLISH, Path.of("src/test/java/com/yahoo/search/significance/model/en.json")); + + significanceModelRegistry = new DefaultSignificanceModelRegistry(map); + searcher = new SignificanceSearcher(significanceModelRegistry); + } + + private Execution createExecution(SignificanceSearcher searcher) { + return new Execution(new Chain<>(searcher), Execution.Context.createContextStub()); + } + + private Execution createExecution() { + return new Execution(new Chain<>(), Execution.Context.createContextStub()); + } + + @Test + void testSignificanceValueOnSimpleQuery() { + + Query q = new Query(); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("Hello", true); + root.addItem(tmp); + tmp = new WordItem("world", true); + root.addItem(tmp); + + q.getModel().getQueryTree().setRoot(root); + + SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get(); + var helloFrequency = model.documentFrequency("Hello"); + var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency()); + + var worldFrequency = model.documentFrequency("world"); + var worldSignificanceValue = SignificanceSearcher.calculateIDF(worldFrequency.corpusSize(), worldFrequency.frequency()); + + Result r = createExecution(searcher).search(q); + + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + WordItem w1 = (WordItem) root.getItem(1); + + assertEquals(helloSignificanceValue, w0.getSignificance()); + assertEquals(worldSignificanceValue, w1.getSignificance()); + + } + + @Test + void testSignificanceValueOnRecursiveQuery() { + Query q = new Query(); + AndItem root = new AndItem(); + WordItem child1 = new WordItem("hello", true); + + AndItem child2 = new AndItem(); + WordItem child2_1 = new WordItem("test", true); + + AndItem child3 = new AndItem(); + AndItem child3_1 = new AndItem(); + WordItem child3_1_1 = new WordItem("usa", true); + + root.addItem(child1); + root.addItem(child2); + root.addItem(child3); + + child2.addItem(child2_1); + child3.addItem(child3_1); + child3_1.addItem(child3_1_1); + + q.getModel().getQueryTree().setRoot(root); + + SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get(); + var helloFrequency = model.documentFrequency("hello"); + var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency()); + + var testFrequency = model.documentFrequency("test"); + var testSignificanceValue = SignificanceSearcher.calculateIDF(testFrequency.corpusSize(), testFrequency.frequency()); + + + + Result r = createExecution(searcher).search(q); + + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + WordItem w1 = (WordItem) ((AndItem) root.getItem(1)).getItem(0); + WordItem w3 = (WordItem) ((AndItem) ((AndItem) root.getItem(2)).getItem(0)).getItem(0); + + assertEquals(helloSignificanceValue, w0.getSignificance()); + assertEquals(testSignificanceValue, w1.getSignificance()); + assertEquals(SignificanceSearcher.calculateIDF(10, 2), w3.getSignificance()); + + } + + @Test + void testSignificanceValueOnEmptyQuery() { + Query q = new Query(); + q.getModel().setLanguage(Language.NORWEGIAN_BOKMAL); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("Hei", true); + root.addItem(tmp); + tmp = new WordItem("Verden", true); + root.addItem(tmp); + + + q.getModel().getQueryTree().setRoot(root); + Result r = createExecution(searcher).search(q); + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + + WordItem w0 = (WordItem) root.getItem(0); + WordItem w1 = (WordItem) root.getItem(1); + + Result r0 = createExecution().search(q); + root = (AndItem) r0.getQuery().getModel().getQueryTree().getRoot(); + + WordItem w0_0 = (WordItem) root.getItem(0); + WordItem w0_1 = (WordItem) root.getItem(1); + + assertEquals(w0_0.getSignificance(), w0.getSignificance()); + assertEquals(w0_1.getSignificance(), w1.getSignificance()); + + } +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index 20ca81234a6..b5e2839c4c0 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -464,7 +464,12 @@ public class VespaSerializerTestCase { @Test void testFuzzyAnnotations() { + parseAndConfirm("foo contains ({maxEditDistance:3}fuzzy(\"a\"))"); parseAndConfirm("foo contains ({maxEditDistance:3,prefixLength:5}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({maxEditDistance:3,prefixLength:5,prefix:true}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({prefixLength:5,prefix:true}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({maxEditDistance:3,prefix:true}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({prefix:true}fuzzy(\"a\"))"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 29a651aabf4..91f5984481a 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -437,23 +437,27 @@ public class YqlParserTestCase { QueryTree x = parse("select foo from bar where baz contains fuzzy(\"a b\")"); Item root = x.getRoot(); assertSame(FuzzyItem.class, root.getClass()); - assertEquals("baz", ((FuzzyItem) root).getIndexName()); - assertEquals("a b", ((FuzzyItem) root).stringValue()); - assertEquals(FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE, ((FuzzyItem) root).getMaxEditDistance()); - assertEquals(FuzzyItem.DEFAULT_PREFIX_LENGTH, ((FuzzyItem) root).getPrefixLength()); + var fuzzy = (FuzzyItem) root; + assertEquals("baz", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE, fuzzy.getMaxEditDistance()); + assertEquals(FuzzyItem.DEFAULT_PREFIX_LENGTH, fuzzy.getPrefixLength()); + assertFalse(fuzzy.isPrefixMatch()); } @Test void testFuzzyAnnotations() { QueryTree x = parse( - "select foo from bar where baz contains ({maxEditDistance: 3, prefixLength: 10}fuzzy(\"a b\"))" + "select foo from bar where baz contains ({maxEditDistance: 3, prefixLength: 10, prefix: true}fuzzy(\"a b\"))" ); Item root = x.getRoot(); assertSame(FuzzyItem.class, root.getClass()); - assertEquals("baz", ((FuzzyItem) root).getIndexName()); - assertEquals("a b", ((FuzzyItem) root).stringValue()); - assertEquals(3, ((FuzzyItem) root).getMaxEditDistance()); - assertEquals(10, ((FuzzyItem) root).getPrefixLength()); + var fuzzy = (FuzzyItem) root; + assertEquals("baz", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(3, fuzzy.getMaxEditDistance()); + assertEquals(10, fuzzy.getPrefixLength()); + assertTrue(fuzzy.isPrefixMatch()); } @Test diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index f4571f04a5d..f863816dab2 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -671,8 +671,39 @@ public class SelectTestCase { QueryTree x = parseWhere("{ \"contains\": [\"description\", { \"fuzzy\": [\"a b\"] }] }"); Item root = x.getRoot(); assertSame(FuzzyItem.class, root.getClass()); - assertEquals("description", ((FuzzyItem) root).getIndexName()); - assertEquals("a b", ((FuzzyItem) root).stringValue()); + var fuzzy = (FuzzyItem) root; + assertEquals("description", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE, fuzzy.getMaxEditDistance()); + assertEquals(FuzzyItem.DEFAULT_PREFIX_LENGTH, fuzzy.getPrefixLength()); + assertFalse(fuzzy.isPrefixMatch()); + } + + @Test + void fuzzy_with_annotations() { + var where = """ + { + "contains": ["description", { + "fuzzy": { + "children": ["a b"], + "attributes": { + "maxEditDistance": 3, + "prefixLength": 10, + "prefix": true + } + } + }] + } + """; + QueryTree x = parseWhere(where); + Item root = x.getRoot(); + assertSame(FuzzyItem.class, root.getClass()); + var fuzzy = (FuzzyItem) root; + assertEquals("description", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(3, fuzzy.getMaxEditDistance()); + assertEquals(10, fuzzy.getPrefixLength()); + assertTrue(fuzzy.isPrefixMatch()); } //------------------------------------------------------------------- grouping tests |