diff options
Diffstat (limited to 'container-search/src')
14 files changed, 566 insertions, 211 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index e20c4271fe0..441c4326355 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -270,7 +270,12 @@ public class ClusterSearcher extends Searcher { Result result = task.get(); mergedResult.mergeWith(result); mergedResult.hits().addAll(result.hits().asUnorderedHits()); - } catch (ExecutionException | InterruptedException e) { + } catch (ExecutionException e) { + mergedResult.hits().addError(ErrorMessage.createInternalServerError("Failed querying '" + + query.getModel().getRestrict() + "': " + + Exceptions.toMessageString(e), + e)); + } catch (InterruptedException e) { mergedResult.hits().addError(ErrorMessage.createInternalServerError("Failed querying '" + query.getModel().getRestrict() + "': " + Exceptions.toMessageString(e))); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index 7a97766e025..30480ce1098 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -120,7 +120,7 @@ public class NormalizingSearcher extends Searcher { List<Alternative> terms = block.getAlternatives(); for (Alternative term : terms) { String accentDropped = linguistics.getTransformer().accentDrop(term.word, language); - if ( ! term.word.equals(accentDropped) && accentDropped.length() > 0) + if ( ! term.word.equals(accentDropped) && !accentDropped.isEmpty()) block.addTerm(accentDropped, term.exactness * .7d); } } @@ -144,15 +144,14 @@ public class NormalizingSearcher extends Searcher { } private void normalizeWord(Language language, IndexFacts.Session indexFacts, TermItem term, ListIterator<Item> i) { - if ( ! (term instanceof WordItem)) return; + if ( ! (term instanceof WordItem word)) return; if ( ! term.isNormalizable()) return; Index index = indexFacts.getIndex(term.getIndexName()); if (index.isAttribute()) return; if ( ! index.getNormalize()) return; - WordItem word = (WordItem) term; String accentDropped = linguistics.getTransformer().accentDrop(word.getWord(), language); - if (accentDropped.length() == 0) + if (accentDropped.isEmpty()) i.remove(); else word.setWord(accentDropped); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java index 708c6de1212..0559bd808bc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java @@ -6,18 +6,20 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.Properties; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; +import java.util.Arrays; import java.util.Map; /** - * A searcher which does parametrized collapsing. + * A searcher which does parameterized collapsing. * * @author Steinar Knutsen */ @@ -30,12 +32,16 @@ public class FieldCollapsingSearcher extends Searcher { private static final CompoundName collapsesize = CompoundName.from("collapsesize"); private static final CompoundName collapseSummaryName = CompoundName.from("collapse.summary"); + /** Separator used for the fieldnames in collapsefield */ + private static final String separator = ","; + /** Maximum number of queries to send next searcher */ private static final int maxQueries = 4; /** * The max number of hits that will be preserved per unique - * value of the collapsing parameter. + * value of the collapsing parameter, + * if no field-specific value is configured. */ private int defaultCollapseSize; @@ -85,11 +91,14 @@ public class FieldCollapsingSearcher extends Searcher { */ @Override public Result search(com.yahoo.search.Query query, Execution execution) { - String collapseField = query.properties().getString(collapsefield); + String collapseFieldParam = query.properties().getString(collapsefield); + + if (collapseFieldParam == null) return execution.search(query); + + String[] collapseFields = collapseFieldParam.split(separator); - if (collapseField == null) return execution.search(query); + int globalCollapseSize = query.properties().getInteger(collapsesize, defaultCollapseSize); - int collapseSize = query.properties().getInteger(collapsesize, defaultCollapseSize); query.properties().set(collapse, "0"); int hitsToRequest = query.getHits() != 0 ? (int) Math.ceil((query.getOffset() + query.getHits() + 1) * extraFactor) : 0; @@ -103,12 +112,15 @@ public class FieldCollapsingSearcher extends Searcher { String collapseSummary = query.properties().getString(collapseSummaryName); String summaryClass = (collapseSummary == null) ? query.getPresentation().getSummary() : collapseSummary; - query.trace("Collapsing by '" + collapseField + "' using summary '" + collapseSummary + "'", 2); + query.trace("Collapsing by '" + Arrays.toString(collapseFields) + "' using summary '" + collapseSummary + "'", 2); do { resultSource = search(query.clone(), execution, nextOffset, hitsToRequest); fill(resultSource, summaryClass, execution); - collapse(result, knownCollapses, resultSource, collapseField, collapseSize); + + collapse(result, knownCollapses, resultSource, + collapseFields, query.properties(), globalCollapseSize + ); hitsAfterCollapse = result.getHitCount(); if (resultSource.getTotalHitCount() < (hitsToRequest + nextOffset)) { @@ -143,35 +155,63 @@ public class FieldCollapsingSearcher extends Searcher { /** * Collapse logic. Preserves only maxHitsPerField hits - * for each unique value of the collapsing parameter. + * for each unique value of the collapsing parameters. + * Uses collapsefields sequentially. */ - private void collapse(Result result, Map<String, Integer> knownCollapses, - Result resultSource, String collapseField, int collapseSize) { + private void collapse(Result result, Map<String, Integer> knownCollapses, Result resultSource, + String[] collapseFields, Properties queryProperties, int globalCollapseSize) { + for (Hit unknownHit : resultSource.hits()) { if (!(unknownHit instanceof FastHit hit)) { result.hits().add(unknownHit); continue; } - Object peek = hit.getField(collapseField); - String collapseId = peek != null ? peek.toString() : null; - if (collapseId == null) { - result.hits().add(hit); - continue; - } - if (knownCollapses.containsKey(collapseId)) { - int numHitsThisField = knownCollapses.get(collapseId); + boolean addHit = true; - if (numHitsThisField < collapseSize) { - result.hits().add(hit); - ++numHitsThisField; - knownCollapses.put(collapseId, numHitsThisField); + for (String collapseField : collapseFields) { + + Object peek = hit.getField(collapseField); + String collapseId = peek != null ? peek.toString() : null; + if (collapseId == null) { + continue; + } + + // prepending the fieldname is necessary to distinguish between values in the different collapsefields + // @ cannot occur in fieldnames + String collapseKey = collapseField + "@" + collapseId; + + if (knownCollapses.containsKey(collapseKey)) { + int numHitsThisField = knownCollapses.get(collapseKey); + int collapseSize = getCollapseSize(queryProperties, collapseField, globalCollapseSize); + + if (numHitsThisField < collapseSize) { + ++numHitsThisField; + knownCollapses.put(collapseKey, numHitsThisField); + } else { + addHit = false; + // immediate return, so that following collapseFields do not record the fieldvalues of this hit + // needed for sequential collapsing, otherwise later collapsefields would remove too many hits + break; + } + } else { + knownCollapses.put(collapseKey, 1); } - } else { - knownCollapses.put(collapseId, 1); + } + + if (addHit) { result.hits().add(hit); } } } + private int getCollapseSize(Properties properties, String fieldName, int globalCollapseSize) { + Integer fieldCollapseSize = properties.getInteger(collapsesize.append(fieldName)); + + if (fieldCollapseSize != null) { + return fieldCollapseSize; + } + + return globalCollapseSize; + } } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index feabf7f76f1..16345c98cff 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -27,30 +27,18 @@ public final class ParserFactory { */ @SuppressWarnings("deprecation") public static Parser newInstance(Query.Type type, ParserEnvironment environment) { - switch (type) { - case ALL: - return new AllParser(environment, false); - case ANY: - return new AnyParser(environment); - case PHRASE: - return new PhraseParser(environment); - case ADVANCED: - return new AdvancedParser(environment); - case WEB: - return new WebParser(environment); - case PROGRAMMATIC: - return new ProgrammaticParser(); - case YQL: - return new YqlParser(environment); - case SELECT: - return new SelectParser(environment); - case WEAKAND: - return new AllParser(environment, true); - case TOKENIZE: - return new TokenizeParser(environment); - default: - throw new UnsupportedOperationException(type.toString()); - } + return switch (type) { + case ALL -> new AllParser(environment, false); + case ANY -> new AnyParser(environment); + case PHRASE -> new PhraseParser(environment); + case ADVANCED -> new AdvancedParser(environment); + case WEB -> new WebParser(environment); + case PROGRAMMATIC -> new ProgrammaticParser(); + case YQL -> new YqlParser(environment); + case SELECT -> new SelectParser(environment); + case WEAKAND -> new AllParser(environment, true); + case TOKENIZE -> new TokenizeParser(environment); + }; } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java index bef766e7ef9..70f6e405a92 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java @@ -15,6 +15,7 @@ public class ConversionContext { private final String destination; private final CompiledQueryProfileRegistry registry; private final Map<String, Embedder> embedders; + private final Map<String, String> contextValues; private final Language language; public ConversionContext(String destination, CompiledQueryProfileRegistry registry, Embedder embedder, @@ -30,6 +31,7 @@ public class ConversionContext { this.embedders = embedders; this.language = context.containsKey("language") ? Language.fromLanguageTag(context.get("language")) : Language.UNKNOWN; + this.contextValues = context; } /** Returns the local name of the field which will receive the converted value (or null when this is empty) */ @@ -44,6 +46,9 @@ public class ConversionContext { /** Returns the language, which is never null but may be UNKNOWN */ Language language() { return language; } + /** Returns a read-only map of context key-values which can be looked up during conversion. */ + Map<String,String> contextValues() { return contextValues; } + /** Returns an empty context */ public static ConversionContext empty() { return new ConversionContext(null, null, Embedder.throwsOnUse.asMap(), Map.of()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index cfadd79de8f..e16f8e7b0cd 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -48,7 +48,8 @@ public class TensorFieldType extends FieldType { @Override public Object convertFrom(Object o, ConversionContext context) { if (o instanceof SubstituteString) return new SubstituteStringTensor((SubstituteString) o, type); - return new TensorConverter(context.embedders()).convertTo(type, context.destination(), o, context.language()); + return new TensorConverter(context.embedders()).convertTo(type, context.destination(), o, + context.language(), context.contextValues()); } public static TensorFieldType fromTypeString(String s) { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java index c9f935e5f52..25a5c277dce 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java @@ -44,7 +44,8 @@ public class RankProfileInputProperties extends Properties { value = tensorConverter.convertTo(expectedType, name.last(), value, - query.getModel().getLanguage()); + query.getModel().getLanguage(), + context); } } catch (IllegalArgumentException e) { diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorMessage.java b/container-search/src/main/java/com/yahoo/search/result/ErrorMessage.java index a6f57aa866a..503bbd725c3 100644 --- a/container-search/src/main/java/com/yahoo/search/result/ErrorMessage.java +++ b/container-search/src/main/java/com/yahoo/search/result/ErrorMessage.java @@ -163,6 +163,14 @@ public class ErrorMessage extends com.yahoo.processing.request.ErrorMessage { return new ErrorMessage(INTERNAL_SERVER_ERROR.code, "Internal server error.", detailedMessage); } + /** + * Creates an error analog to HTTP internal server error. If this error is present, a + * HTTP layer will return 500. + */ + public static ErrorMessage createInternalServerError(String detailedMessage, Throwable cause) { + return new ErrorMessage(INTERNAL_SERVER_ERROR.code, "Internal server error.", detailedMessage, cause); + } + /** Wraps an error message received in a SearchReply packet */ public static ErrorMessage createSearchReplyError(String detailedMessage) { return new ErrorMessage(RESULT_HAS_ERRORS.code, "Error in search reply.", detailedMessage); diff --git a/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java b/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java index 6da53ae699c..94f92c7fd48 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java +++ b/container-search/src/main/java/com/yahoo/search/schema/internal/TensorConverter.java @@ -19,7 +19,8 @@ import java.util.regex.Pattern; */ public class TensorConverter { - private static final Pattern embedderArgumentRegexp = Pattern.compile("^([A-Za-z0-9_\\-.]+),\\s*([\"'].*[\"'])"); + private static final Pattern embedderArgumentAndQuotedTextRegexp = Pattern.compile("^([A-Za-z0-9_@\\-.]+),\\s*([\"'].*[\"'])"); + private static final Pattern embedderArgumentAndReferenceRegexp = Pattern.compile("^([A-Za-z0-9_@\\-.]+),\\s*(@.*)"); private final Map<String, Embedder> embedders; @@ -27,8 +28,9 @@ public class TensorConverter { this.embedders = embedders; } - public Tensor convertTo(TensorType type, String key, Object value, Language language) { - var context = new Embedder.Context(key).setLanguage(language); + public Tensor convertTo(TensorType type, String key, Object value, Language language, + Map<String, String> contextValues) { + var context = new Embedder.Context(key).setLanguage(language).setContextValues(contextValues); Tensor tensor = toTensor(type, value, context); if (tensor == null) return null; if (! tensor.type().isAssignableTo(type)) @@ -55,16 +57,16 @@ public class TensorConverter { String embedderId; // Check if arguments specifies an embedder with the format embed(embedder, "text to encode") - Matcher matcher = embedderArgumentRegexp.matcher(argument); - if (matcher.matches()) { + Matcher matcher; + if (( matcher = embedderArgumentAndQuotedTextRegexp.matcher(argument)).matches()) { embedderId = matcher.group(1); + embedder = requireEmbedder(embedderId); argument = matcher.group(2); - if ( ! embedders.containsKey(embedderId)) { - throw new IllegalArgumentException("Can't find embedder '" + embedderId + "'. " + - "Valid embedders are " + validEmbedders(embedders)); - } - embedder = embedders.get(embedderId); - } else if (embedders.size() == 0) { + } else if (( matcher = embedderArgumentAndReferenceRegexp.matcher(argument)).matches()) { + embedderId = matcher.group(1); + embedder = requireEmbedder(embedderId); + argument = matcher.group(2); + } else if (embedders.isEmpty()) { throw new IllegalStateException("No embedders provided"); // should never happen } else if (embedders.size() > 1) { throw new IllegalArgumentException("Multiple embedders are provided but no embedder id is given. " + @@ -74,19 +76,35 @@ public class TensorConverter { embedderId = entry.getKey(); embedder = entry.getValue(); } - return embedder.embed(removeQuotes(argument), embedderContext.copy().setEmbedderId(embedderId), type); + return embedder.embed(resolve(argument, embedderContext), embedderContext.copy().setEmbedderId(embedderId), type); } - private static String removeQuotes(String s) { - if (s.startsWith("'") && s.endsWith("'")) { + private Embedder requireEmbedder(String embedderId) { + if ( ! embedders.containsKey(embedderId)) + throw new IllegalArgumentException("Can't find embedder '" + embedderId + "'. " + + "Valid embedders are " + validEmbedders(embedders)); + return embedders.get(embedderId); + } + + private static String resolve(String s, Embedder.Context embedderContext) { + if (s.startsWith("'") && s.endsWith("'")) return s.substring(1, s.length() - 1); - } - if (s.startsWith("\"") && s.endsWith("\"")) { + if (s.startsWith("\"") && s.endsWith("\"")) return s.substring(1, s.length() - 1); - } + if (s.startsWith("@")) + return resolveReference(s, embedderContext); return s; } + private static String resolveReference(String s, Embedder.Context embedderContext) { + String referenceKey = s.substring(1); + String referencedValue = embedderContext.getContextValues().get(referenceKey); + if (referencedValue == null) + throw new IllegalArgumentException("Could not resolve query parameter reference '" + referenceKey + + "' used in an embed() argument"); + return referencedValue; + } + private static String validEmbedders(Map<String, Embedder> embedders) { List<String> embedderIds = new ArrayList<>(); embedders.forEach((key, value) -> embedderIds.add(key)); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 29f2d9aff9a..7ae02c18e7a 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -595,7 +595,7 @@ public class YqlParser implements Parser { } WandItem out = new WandItem(getIndex(args.get(0)), targetNumHits); Double scoreThreshold = getAnnotation(ast, SCORE_THRESHOLD, Double.class, null, - "min score for hit inclusion"); + "score must be above this threshold for hit inclusion"); if (scoreThreshold != null) { out.setScoreThreshold(scoreThreshold); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java index 361079f7595..d45d3866783 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -68,6 +68,74 @@ public class FieldCollapsingSearcherTestCase { assertEquals(1, checker.queryCount); } + /** + * Tests that we do not fail on documents with missing collapsefield + * and that they are kept in the result. + */ + @Test + void testFieldCollapsingWithCollapseFieldMissing() { + Map<Searcher, Searcher> chained = new HashMap<>(); + + // Set up + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); + r.hits().add(createHitWithoutFields("http://acme.org/a.html", 10)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 1)); + r.hits().add(createHitWithoutFields("http://acme.org/c.html", 9)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 2)); + r.hits().add(createHitAmid("http://acme.org/d.html", 7, 2)); + r.setTotalHitCount(5); + docsource.addResult(q, r); + + // Test basic collapsing on amid + q = new Query("?query=test_collapse&collapsefield=amid&collapsesize=1"); + r = doSearch(collapse, q, 0, 10, chained); + + assertEquals(4, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + + assertHitWithoutFields("http://acme.org/a.html", 10, r.hits().get(0)); + assertHitAmid("http://acme.org/b.html", 9, 1, r.hits().get(1)); + assertHitWithoutFields("http://acme.org/c.html", 9, r.hits().get(2)); + assertHitAmid("http://acme.org/d.html", 8, 2, r.hits().get(3)); + } + + @Test + void testFieldCollapsingOnMultipleFieldsWithCollapseFieldsMissing() { + Map<Searcher, Searcher> chained = new HashMap<>(); + + // Set up + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); + r.hits().add(createHitWithoutFields("http://acme.org/a.html", 10)); // - - + r.hits().add(createHitBmid("http://acme.org/b.html", 9, 1)); // - 1 + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 1)); // 1 - + r.hits().add(createHitBmid("http://acme.org/d.html", 8, 1)); // - 1 + r.hits().add(createHit("http://acme.org/e.html", 8, 2, 2)); // 2 2 + r.setTotalHitCount(5); + docsource.addResult(q, r); + + // Test basic collapsing + q = new Query("?query=test_collapse&collapsefield=amid,bmid&collapsesize=1"); + r = doSearch(collapse, q, 0, 10, chained); + + assertEquals(4, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + + assertHitWithoutFields("http://acme.org/a.html", 10, r.hits().get(0)); + assertHitBmid("http://acme.org/b.html", 9, 1, r.hits().get(1)); + assertHitAmid("http://acme.org/c.html", 9, 1, r.hits().get(2)); + assertHit("http://acme.org/e.html", 8, 2, 2, r.hits().get(3)); + } + @Test void testFieldCollapsing() { Map<Searcher, Searcher> chained = new HashMap<>(); @@ -77,20 +145,16 @@ public class FieldCollapsingSearcherTestCase { DocumentSourceSearcher docsource = new DocumentSourceSearcher(); chained.put(collapse, docsource); - // Caveat: Collapse is set to false, because that's what the - // collapser asks for - Query q = new Query("?query=test_collapse&collapsefield=amid"); - // The searcher turns off collapsing further on in the chain - q.properties().set("collapse", "0"); + Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 1)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 1)); + r.hits().add(createHitAmid("http://acme.org/e.html", 8, 2)); + r.hits().add(createHitAmid("http://acme.org/f.html", 7, 2)); + r.hits().add(createHitAmid("http://acme.org/g.html", 7, 3)); + r.hits().add(createHitAmid("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -100,46 +164,47 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html", 10, 0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); + assertHitAmid("http://acme.org/a.html", 10, 0, r.hits().get(0)); + assertHitAmid("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHitAmid("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHitAmid("http://acme.org/g.html", 7, 3, r.hits().get(3)); } + /** + * Test that collapsing works if multiple searches are necessary. + */ @Test void testFieldCollapsingTwoPhase() { - // Set up Map<Searcher, Searcher> chained = new HashMap<>(); - FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); + + // Set up + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(1, 1.0); DocumentSourceSearcher docsource = new DocumentSourceSearcher(); chained.put(collapse, docsource); - // Caveat: Collapse is set to false, because that's what the - // collapser asks for - Query q = new Query("?query=test_collapse&collapsefield=amid"); - // The searcher turns off collapsing further on in the chain - q.properties().set("collapse", "0"); + + Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 1)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 1)); + r.hits().add(createHitAmid("http://acme.org/e.html", 8, 2)); + r.hits().add(createHitAmid("http://acme.org/f.html", 7, 2)); + r.hits().add(createHitAmid("http://acme.org/g.html", 7, 3)); + r.hits().add(createHitAmid("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); // Test basic collapsing on mid q = new Query("?query=test_collapse&collapsefield=amid"); - r = doSearch(collapse, q, 0, 10, chained); + r = doSearch(collapse, q, 0, 4, chained); assertEquals(4, r.getHitCount()); - assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html", 10, 0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); + assertEquals(2, docsource.getQueryCount()); + assertHitAmid("http://acme.org/a.html", 10, 0, r.hits().get(0)); + assertHitAmid("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHitAmid("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHitAmid("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -152,14 +217,14 @@ public class FieldCollapsingSearcherTestCase { Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 1)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 1)); + r.hits().add(createHitAmid("http://acme.org/e.html", 8, 2)); + r.hits().add(createHitAmid("http://acme.org/f.html", 7, 2)); + r.hits().add(createHitAmid("http://acme.org/g.html", 7, 3)); + r.hits().add(createHitAmid("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -183,20 +248,18 @@ public class FieldCollapsingSearcherTestCase { DocumentSourceSearcher docsource = new DocumentSourceSearcher(); chained.put(collapse, docsource); - Query q = new Query("?query=test_collapse&collapsesize=1&collapsefield=amid"); - // The searcher turns off collapsing further on in the chain - q.properties().set("collapse", "0"); + Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 0)); - r.hits().add(createHit("http://acme.org/d.html", 8, 0)); - r.hits().add(createHit("http://acme.org/e.html", 8, 0)); - r.hits().add(createHit("http://acme.org/f.html", 7, 0)); - r.hits().add(createHit("http://acme.org/g.html", 7, 0)); - r.hits().add(createHit("http://acme.org/h.html", 6, 0)); - r.hits().add(createHit("http://acme.org/i.html", 5, 1)); - r.hits().add(createHit("http://acme.org/j.html", 4, 2)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 0)); + r.hits().add(createHitAmid("http://acme.org/e.html", 8, 0)); + r.hits().add(createHitAmid("http://acme.org/f.html", 7, 0)); + r.hits().add(createHitAmid("http://acme.org/g.html", 7, 0)); + r.hits().add(createHitAmid("http://acme.org/h.html", 6, 0)); + r.hits().add(createHitAmid("http://acme.org/i.html", 5, 1)); + r.hits().add(createHitAmid("http://acme.org/j.html", 4, 2)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -206,15 +269,171 @@ public class FieldCollapsingSearcherTestCase { assertEquals(2, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/a.html", 10, 0, r.hits().get(0)); - assertHit("http://acme.org/i.html", 5, 1, r.hits().get(1)); + assertHitAmid("http://acme.org/a.html", 10, 0, r.hits().get(0)); + assertHitAmid("http://acme.org/i.html", 5, 1, r.hits().get(1)); // Next results docsource.resetQueryCount(); r = doSearch(collapse, q, 2, 2, chained); assertEquals(1, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/j.html", 4, 2, r.hits().get(0)); + assertHitAmid("http://acme.org/j.html", 4, 2, r.hits().get(0)); + } + + /** + * Tests that collapsing hits with 2 fields works, + * this test also shows that field order is important + */ + @Test + void testCollapsingWithMultipleFields() { + // Set up + Map<Searcher, Searcher> chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html", 10, 1, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 1, 1)); + r.hits().add(createHit("http://acme.org/c.html", 8, 0, 1)); + r.hits().add(createHit("http://acme.org/d.html", 7, 1, 0)); + r.setTotalHitCount(4); + docsource.addResult(q, r); + + // Test collapsing, starting with amid + q = new Query("?query=test_collapse&collapsesize=1&collapsefield=amid,bmid"); + r = doSearch(collapse, q, 0, 4, chained); + + assertEquals(2, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertHit("http://acme.org/a.html", 10, 1, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 8, 0, 1, r.hits().get(1)); + + docsource.resetQueryCount(); + + // Test collapsing, starting with bmid + q = new Query("?query=test_collapse&collapsesize=1&collapsefield=bmid,amid"); + r = doSearch(collapse, q, 0, 4, chained); + + assertEquals(1, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertHit("http://acme.org/a.html", 10, 1, 0, r.hits().get(0)); + } + + /** + * Tests that using different collapse sizes for different fields works + */ + @Test + void testCollapsingWithMultipleFieldsAndMultipleCollapseSizes() { + // Set up + Map<Searcher, Searcher> chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html", 10, 1, 1)); + r.hits().add(createHit("http://acme.org/b.html", 9, 1, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 0, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1, 0)); + r.setTotalHitCount(4); + docsource.addResult(q, r); + + // Test collapsing + // default collapsesize is used for amid, bmid is set to 2 + q = new Query("?query=test_collapse&collapsefield=amid,bmid&collapsesize.bmid=2"); + r = doSearch(collapse, q, 0, 4, chained); + + assertEquals(2, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertHit("http://acme.org/a.html", 10, 1, 1, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 0, 1, r.hits().get(1)); + } + + /** + * Tests that using different collapse sizes for different fields works, + * test that the different ways to configure collapse size have the correct precedence + */ + @Test + void testCollapsingWithMultipleFieldsAndMultipleCollapseSizeSources() { + // Set up + Map<Searcher, Searcher> chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html", 10, 1, 1)); + r.hits().add(createHit("http://acme.org/b.html", 9, 1, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 0, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1, 0)); + r.hits().add(createHit("http://acme.org/3.html", 8, 1, 0)); + r.setTotalHitCount(5); + docsource.addResult(q, r); + + // Test collapsing + // collapsesize 10 overwrites the default for amid & bmid + // collapsize.bmid overwrites the collapsesize for bmid again + q = new Query("?query=test_collapse&collapsesize=10&collapsefield=amid,bmid&collapsesize.bmid=2"); + r = doSearch(collapse, q, 0, 5, chained); + + assertEquals(4, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertHit("http://acme.org/a.html", 10, 1, 1, r.hits().get(0)); + assertHit("http://acme.org/b.html", 9, 1, 0, r.hits().get(1)); + assertHit("http://acme.org/c.html", 9, 0, 1, r.hits().get(2)); + assertHit("http://acme.org/d.html", 8, 1, 0, r.hits().get(3)); + } + + /** + * Tests that collapsing on multiple fields works if we have to search multiple + * time to get enough hits + */ + @Test + void testCollapsingOnMoreFieldsWithManySimilarFieldValues() { + // Set up + Map<Searcher, Searcher> chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(4, 1.0); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html", 10, 0, 1, 1)); // first hit + r.hits().add(createHit("http://acme.org/b.html", 9, 0, 1, 2)); + r.hits().add(createHit("http://acme.org/c.html", 9, 0, 6, 2)); // - - 1. search: 1 + r.hits().add(createHit("http://acme.org/d.html", 8, 0, 6, 3)); + r.hits().add(createHit("http://acme.org/e.html", 8, 0, 6, 3)); + r.hits().add(createHit("http://acme.org/f.html", 7, 0, 6, 3)); // - - 1. search: 2 + r.hits().add(createHit("http://acme.org/g.html", 7, 0, 1, 1)); + r.hits().add(createHit("http://acme.org/h.html", 6, 1, 1, 1)); + r.hits().add(createHit("http://acme.org/i.html", 5, 2, 2, 1)); // - - 1. search: 3 + r.hits().add(createHit("http://acme.org/j.html", 4, 3, 3, 2)); // 3rd hit, cmid new + r.hits().add(createHit("http://acme.org/k.html", 4, 3, 4, 3)); + r.hits().add(createHit("http://acme.org/l.html", 4, 3, 5, 3)); // - - 1. search: 4 + r.hits().add(createHit("http://acme.org/m.html", 4, 4, 6, 3)); // 4th hit, amid new + r.hits().add(createHit("http://acme.org/n.html", 4, 4, 7, 4)); + r.setTotalHitCount(14); + docsource.addResult(q, r); + + // Test collapsing + q = new Query("?query=test_collapse&collapsesize=1&collapsefield=amid,bmid,cmid"); + r = doSearch(collapse, q, 0, 2, chained); + + assertEquals(2, r.getHitCount()); + assertEquals(4, docsource.getQueryCount()); + assertHit("http://acme.org/a.html", 10, 0, 1, 1, r.hits().get(0)); + assertHit("http://acme.org/j.html", 4, 3, 3, 2, r.hits().get(1)); + + // Next results + docsource.resetQueryCount(); + r = doSearch(collapse, q, 2, 2, chained); + assertEquals(1, r.getHitCount()); + assertEquals(3, docsource.getQueryCount()); + assertHit("http://acme.org/m.html", 4, 4, 6, 3, r.hits().get(0)); } /** @@ -228,20 +447,18 @@ public class FieldCollapsingSearcherTestCase { DocumentSourceSearcher docsource = new DocumentSourceSearcher(); chained.put(collapse, docsource); - Query q = new Query("?query=test_collapse&collapse=true&collapsefield=amid"); - // The searcher turns off collapsing further on in the chain - q.properties().set("collapse", "0"); + Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 1)); - r.hits().add(createHit("http://acme.org/b.html", 10, 1)); - r.hits().add(createHit("http://acme.org/c.html", 10, 0)); - r.hits().add(createHit("http://acme.org/d.html", 10, 0)); - r.hits().add(createHit("http://acme.org/e.html", 10, 0)); - r.hits().add(createHit("http://acme.org/f.html", 10, 0)); - r.hits().add(createHit("http://acme.org/g.html", 10, 0)); - r.hits().add(createHit("http://acme.org/h.html", 10, 0)); - r.hits().add(createHit("http://acme.org/i.html", 10, 0)); - r.hits().add(createHit("http://acme.org/j.html", 10, 1)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 1)); + r.hits().add(createHitAmid("http://acme.org/b.html", 10, 1)); + r.hits().add(createHitAmid("http://acme.org/c.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/d.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/e.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/f.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/g.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/h.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/i.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/j.html", 10, 1)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -250,8 +467,8 @@ public class FieldCollapsingSearcherTestCase { r = doSearch(collapse, q, 0, 3, chained); assertEquals(2, r.getHitCount()); - assertHit("http://acme.org/a.html", 10, 1, r.hits().get(0)); - assertHit("http://acme.org/c.html", 10, 0, r.hits().get(1)); + assertHitAmid("http://acme.org/a.html", 10, 1, r.hits().get(0)); + assertHitAmid("http://acme.org/c.html", 10, 0, r.hits().get(1)); } @Test @@ -265,20 +482,16 @@ public class FieldCollapsingSearcherTestCase { chained.put(collapse, messUp); chained.put(messUp, docsource); - // Caveat: Collapse is set to false, because that's what the collapser asks for Query q = new Query("?query=%22test%20collapse%22+b&collapsefield=amid&type=all"); - - // The searcher turns off collapsing further on in the chain - q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 0)); - r.hits().add(createHit("http://acme.org/d.html", 8, 0)); - r.hits().add(createHit("http://acme.org/e.html", 8, 0)); - r.hits().add(createHit("http://acme.org/f.html", 7, 0)); - r.hits().add(createHit("http://acme.org/g.html", 7, 0)); - r.hits().add(createHit("http://acme.org/h.html", 6, 1)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 0)); + r.hits().add(createHitAmid("http://acme.org/e.html", 8, 0)); + r.hits().add(createHitAmid("http://acme.org/f.html", 7, 0)); + r.hits().add(createHitAmid("http://acme.org/g.html", 7, 0)); + r.hits().add(createHitAmid("http://acme.org/h.html", 6, 1)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -288,8 +501,8 @@ public class FieldCollapsingSearcherTestCase { assertEquals(2, docsource.getQueryCount()); assertEquals(2, r.getHitCount()); - assertHit("http://acme.org/a.html", 10, 0, r.hits().get(0)); - assertHit("http://acme.org/h.html", 6, 1, r.hits().get(1)); + assertHitAmid("http://acme.org/a.html", 10, 0, r.hits().get(0)); + assertHitAmid("http://acme.org/h.html", 6, 1, r.hits().get(1)); } @Test @@ -299,20 +512,17 @@ public class FieldCollapsingSearcherTestCase { FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); DocumentSourceSearcher docsource = new DocumentSourceSearcher(); chained.put(collapse, docsource); - // Caveat: Collapse is set to false, because that's what the - // collapser asks for - Query q = new Query("?query=test_collapse&collapsefield=amid&summary=placeholder"); - // The searcher turns off collapsing further on in the chain - q.properties().set("collapse", "0"); + + Query q = new Query("?query=test_collapse&summary=placeholder"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 1)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 1)); + r.hits().add(createHitAmid("http://acme.org/e.html", 8, 2)); + r.hits().add(createHitAmid("http://acme.org/f.html", 7, 2)); + r.hits().add(createHitAmid("http://acme.org/g.html", 7, 3)); + r.hits().add(createHitAmid("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -323,10 +533,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); assertTrue(r.isFilled("placeholder")); - assertHit("http://acme.org/a.html", 10, 0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); + assertHitAmid("http://acme.org/a.html", 10, 0, r.hits().get(0)); + assertHitAmid("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHitAmid("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHitAmid("http://acme.org/g.html", 7, 3, r.hits().get(3)); docsource.resetQueryCount(); // Test basic collapsing on mid @@ -337,10 +547,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(1, docsource.getQueryCount()); assertFalse(r.isFilled("placeholder")); assertTrue(r.isFilled("short")); - assertHit("http://acme.org/a.html", 10, 0, r.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); + assertHitAmid("http://acme.org/a.html", 10, 0, r.hits().get(0)); + assertHitAmid("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHitAmid("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHitAmid("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -350,20 +560,17 @@ public class FieldCollapsingSearcherTestCase { DocumentSourceSearcher docsource = new DocumentSourceSearcher(); Chain<Searcher> chain = new Chain<>(collapse, new AddAggregationStyleGroupingResultSearcher(), docsource); - // Caveat: Collapse is set to false, because that's what the - // collapser asks for - Query q = new Query("?query=test_collapse&collapsefield=amid"); - // The searcher turns off collapsing further on in the chain - q.properties().set("collapse", "0"); + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html", 10, 0)); - r.hits().add(createHit("http://acme.org/b.html", 9, 0)); - r.hits().add(createHit("http://acme.org/c.html", 9, 1)); - r.hits().add(createHit("http://acme.org/d.html", 8, 1)); - r.hits().add(createHit("http://acme.org/e.html", 8, 2)); - r.hits().add(createHit("http://acme.org/f.html", 7, 2)); - r.hits().add(createHit("http://acme.org/g.html", 7, 3)); - r.hits().add(createHit("http://acme.org/h.html", 6, 3)); + r.hits().add(createHitAmid("http://acme.org/a.html", 10, 0)); + r.hits().add(createHitAmid("http://acme.org/b.html", 9, 0)); + r.hits().add(createHitAmid("http://acme.org/c.html", 9, 1)); + r.hits().add(createHitAmid("http://acme.org/d.html", 8, 1)); + r.hits().add(createHitAmid("http://acme.org/e.html", 8, 2)); + r.hits().add(createHitAmid("http://acme.org/f.html", 7, 2)); + r.hits().add(createHitAmid("http://acme.org/g.html", 7, 3)); + r.hits().add(createHitAmid("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -374,10 +581,10 @@ public class FieldCollapsingSearcherTestCase { // Assert that the regular hits are collapsed assertEquals(4 + 1, result.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html", 10, 0, result.hits().get(0)); - assertHit("http://acme.org/c.html", 9, 1, result.hits().get(1)); - assertHit("http://acme.org/e.html", 8, 2, result.hits().get(2)); - assertHit("http://acme.org/g.html", 7, 3, result.hits().get(3)); + assertHitAmid("http://acme.org/a.html", 10, 0, result.hits().get(0)); + assertHitAmid("http://acme.org/c.html", 9, 1, result.hits().get(1)); + assertHitAmid("http://acme.org/e.html", 8, 2, result.hits().get(2)); + assertHitAmid("http://acme.org/g.html", 7, 3, result.hits().get(3)); // Assert that the aggregation group hierarchy is left intact HitGroup root = getFirstGroupIn(result.hits()); @@ -438,16 +645,64 @@ public class FieldCollapsingSearcherTestCase { } } - private FastHit createHit(String uri,int relevancy,int mid) { + private FastHit createHitWithoutFields(String uri, int relevancy) { + return new FastHit(uri,relevancy); + } + + private FastHit createHitAmid(String uri,int relevancy,int amid) { FastHit hit = new FastHit(uri,relevancy); - hit.setField("amid", String.valueOf(mid)); + hit.setField("amid", String.valueOf(amid)); return hit; } - private void assertHit(String uri,int relevancy,int mid,Hit hit) { + private FastHit createHitBmid(String uri,int relevancy,int bmid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("bmid", String.valueOf(bmid)); + return hit; + } + + private FastHit createHit(String uri,int relevancy,int amid,int bmid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("amid", String.valueOf(amid)); + hit.setField("bmid", String.valueOf(bmid)); + return hit; + } + + private FastHit createHit(String uri,int relevancy,int amid,int bmid,int cmid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("amid", String.valueOf(amid)); + hit.setField("bmid", String.valueOf(bmid)); + hit.setField("cmid", String.valueOf(cmid)); + return hit; + } + + private void assertHitWithoutFields(String uri,int relevancy,Hit hit) { assertEquals(uri,hit.getId().toString()); assertEquals(relevancy, ((int) hit.getRelevance().getScore())); - assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); + assertTrue(hit.fields().isEmpty()); + } + + private void assertHitAmid(String uri, int relevancy, int amid, Hit hit) { + assertEquals(uri,hit.getId().toString()); + assertEquals(relevancy, ((int) hit.getRelevance().getScore())); + assertEquals(amid,Integer.parseInt((String) hit.getField("amid"))); + } + + private void assertHitBmid(String uri, int relevancy, int bmid, Hit hit) { + assertEquals(uri,hit.getId().toString()); + assertEquals(relevancy, ((int) hit.getRelevance().getScore())); + assertEquals(bmid,Integer.parseInt((String) hit.getField("bmid"))); + } + + private void assertHit(String uri,int relevancy,int amid,int bmid,Hit hit) { + assertHitAmid(uri,relevancy,amid,hit); + assertEquals(bmid,Integer.parseInt((String) hit.getField("bmid"))); + } + + private void assertHit(String uri,int relevancy,int amid,int bmid,int cmid,Hit hit) { + assertHitAmid(uri,relevancy,amid,hit); + assertHitBmid(uri,relevancy,bmid,hit); + assertEquals(cmid,Integer.parseInt((String) hit.getField("cmid"))); } private static class ZeroHitsControl extends com.yahoo.search.Searcher { diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java index 90e21e5f3b0..429b8d1c6cb 100644 --- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -185,6 +185,21 @@ public class RankProfileInputTest { assertEmbedQuery("embed(emb2, '" + text + "')", embedding2, embedders, Language.UNKNOWN.languageCode()); } + @Test + void testUnembeddedTensorRankFeatureInRequestReferencedFromAParameter() { + String text = "text to embed into a tensor"; + Tensor embedding1 = Tensor.from("tensor<float>(x[5]):[3,7,4,0,0]]"); + + Map<String, Embedder> embedders = Map.of( + "emb1", new MockEmbedder(text, Language.UNKNOWN, embedding1) + ); + assertEmbedQuery("embed(@param1)", embedding1, embedders, null, text); + assertEmbedQuery("embed(emb1, @param1)", embedding1, embedders, null, text); + assertEmbedQueryFails("embed(emb1, @noSuchParam)", embedding1, embedders, + "Could not resolve query parameter reference 'noSuchParam' " + + "used in an embed() argument"); + } + private Query createTensor1Query(String tensorString, String profile, String additionalParams) { return new Query.Builder() .setSchemaInfo(createSchemaInfo()) @@ -202,18 +217,24 @@ public class RankProfileInputTest { } private void assertEmbedQuery(String embed, Tensor expected, Map<String, Embedder> embedders) { - assertEmbedQuery(embed, expected, embedders, null); + assertEmbedQuery(embed, expected, embedders, null, null); } private void assertEmbedQuery(String embed, Tensor expected, Map<String, Embedder> embedders, String language) { + assertEmbedQuery(embed, expected, embedders, language, null); + } + private void assertEmbedQuery(String embed, Tensor expected, Map<String, Embedder> embedders, String language, String param1Value) { String languageParam = language == null ? "" : "&language=" + language; + String param1 = param1Value == null ? "" : "¶m1=" + urlEncode(param1Value); + String destination = "query(myTensor4)"; Query query = new Query.Builder().setRequest(HttpRequest.createTestRequest( "?" + urlEncode("ranking.features." + destination) + "=" + urlEncode(embed) + "&ranking=commonProfile" + - languageParam, + languageParam + + param1, com.yahoo.jdisc.http.HttpRequest.Method.GET)) .setSchemaInfo(createSchemaInfo()) .setQueryProfile(createQueryProfile()) @@ -230,7 +251,7 @@ public class RankProfileInputTest { if (t.getMessage().equals(errMsg)) return; t = t.getCause(); } - fail("Error '" + errMsg + "' not thrown"); + fail("Exception with message '" + errMsg + "' not thrown"); } private CompiledQueryProfile createQueryProfile() { diff --git a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java index 9f244b16139..49eaa9b3a89 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java @@ -82,7 +82,7 @@ public class SortingTestCase { private void requireThatChineseHasCorrectRules(Collator col) { final int reorderCodes [] = {UScript.HAN}; assertEquals("15.1.0.0", col.getUCAVersion().toString()); - assertEquals("153.121.44.0", col.getVersion().toString()); + assertEquals("153.121.44.8", col.getVersion().toString()); assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); assertNotEquals("", ((RuleBasedCollator) col).getRules()); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 783a0ec61de..75e9525f09b 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -45,8 +45,10 @@ import com.yahoo.search.query.Sorting.LowerCaseSorter; import com.yahoo.search.query.Sorting.Order; import com.yahoo.search.query.Sorting.UcaSorter; import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -1210,6 +1212,18 @@ public class YqlParserTestCase { "and string fields. The fieldset mixed has both")); } + // TODO: Put this in the documentation + @Test + public void testProgrammaticYqlParsing() { + Execution execution = new Execution(Execution.Context.createContextStub()); + Parser parser = ParserFactory.newInstance(Query.Type.YQL, + ParserEnvironment.fromExecutionContext(execution.context())); + Query query = new Query(); + query.getModel().setType(Query.Type.YQL); + query.getModel().setQueryString("select * from myDoc where foo contains 'bar' and fuz contains '3'"); + parser.parse(Parsable.fromQueryModel(query.getModel())); + } + private static void assertNumericInItem(String field, long[] values, QueryTree query) { var exp = buildNumericInItem(field, values); assertEquals(exp, query.getRoot()); |