diff options
Diffstat (limited to 'container-search')
23 files changed, 245 insertions, 281 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index a094be943a2..23cdff15ad9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.ObjectTraverser; import com.yahoo.document.GlobalId; import com.yahoo.net.URI; +import com.yahoo.search.dispatch.LeanHit; import com.yahoo.search.query.Sorting; import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; @@ -11,7 +12,6 @@ import com.yahoo.search.result.Relevance; import com.yahoo.data.access.Inspector; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -31,18 +31,18 @@ import java.util.function.BiConsumer; */ public class FastHit extends Hit { - private static final byte [] emptyGID = new byte[GlobalId.LENGTH]; + private static final byte[] emptyGID = new byte[GlobalId.LENGTH]; /** The index of the content node this hit originated at */ - private int distributionKey = 0; + private int distributionKey; /** The local identifier of the content store for this hit on the node it originated at */ - private int partId; + private final int partId; /** The global id of this document in the backend node which produced it */ - private byte [] globalId; + private byte[] globalId; private transient byte[] sortData = null; - // TODO I suspect this one can be dropped. + // TODO: I suspect this one can be dropped. private transient Sorting sortDataSorting = null; /** @@ -73,10 +73,10 @@ public class FastHit extends Hit { distributionKey = 0; } - public FastHit(byte [] gid, double relevance, int partId, int distributionKey) { + public FastHit(byte[] gid, double relevance, int partId, int distributionKey) { this(gid, new Relevance(relevance), partId, distributionKey); } - public FastHit(byte [] gid, Relevance relevance, int partId, int distributionKey) { + public FastHit(byte[] gid, Relevance relevance, int partId, int distributionKey) { super(relevance); this.globalId = gid; this.partId = partId; @@ -109,8 +109,8 @@ public class FastHit extends Hit { */ @Override public URI getId() { - URI uri = super.getId(); - if (uri != null) return uri; + URI id = super.getId(); + if (id != null) return id; // Fallback to index:[source]/[partid]/[id] StringBuilder sb = new StringBuilder(64); @@ -129,16 +129,6 @@ public class FastHit extends Hit { public int getPartId() { return partId; } - /** - * Sets the part id number, which specifies the node where this hit is - * found. The row count is used to decode the part id into a column and a - * row number: the number of n least significant bits required to hold the - * highest row number are the row bits, the rest are column bits. - * - * Note: Remove partId when all dispatching happens from the container dispatcher, not fdispatch - */ - public void setPartId(int partId) { this.partId = partId; } - /** Returns the index of the node this hit originated at */ public int getDistributionKey() { return distributionKey; } @@ -167,17 +157,7 @@ public class FastHit extends Hit { if (!left.hasSortData(sorting) || !right.hasSortData(sorting)) { return 0; // cannot sort } - int i = Arrays.mismatch(left.sortData, right.sortData); - if (i < 0) { - return 0; - } - int max = Integer.min(left.sortData.length, right.sortData.length); - if (i >= max) { - return left.sortData.length - right.sortData.length; - } - int vl = (int) left.sortData[i] & 0xFF; - int vr = (int) right.sortData[i] & 0xFF; - return vl - vr; + return LeanHit.compareData(left.sortData, right.sortData); } /** For internal use */ @@ -188,11 +168,6 @@ public class FastHit extends Hit { summaries.add(0, new SummaryData(this, docsumDef, value, 1 + summaries.size())); } - /** Returns the raw summary data available in this as an unmodifiable list */ - public List<SummaryData> summaryData() { - return Collections.unmodifiableList(summaries); - } - /** * Returns values for the features listed in * <a href="https://docs.vespa.ai/en/reference/schema-reference.html#summary-features">summary-features</a> diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java index 6067f85df9b..722e7155dc8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java @@ -5,7 +5,6 @@ import com.yahoo.search.query.Sorting; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; -import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -18,15 +17,14 @@ public class SortDataHitSorter { return; } var fallbackComparator = fallbackOrderer.getComparator(); - Collections.sort(hits, getComparator(sorting, fallbackComparator)); + hits.sort(getComparator(sorting, fallbackComparator)); } public static boolean isSortable(Hit hit, Sorting sorting) { if (sorting == null) { return false; } - if (hit instanceof FastHit) { - var fhit = (FastHit) hit; + if (hit instanceof FastHit fhit) { return fhit.hasSortData(sorting); } else { return false; @@ -42,20 +40,14 @@ public class SortDataHitSorter { } private static int compareTwo(Hit left, Hit right, Sorting sorting) { - if (left == null || right == null || !(left instanceof FastHit) || !(right instanceof FastHit)) { - return 0; - } - FastHit fl = (FastHit) left; - FastHit fr = (FastHit) right; + if (!(left instanceof FastHit fl) || !(right instanceof FastHit fr)) return 0; return FastHit.compareSortData(fl, fr, sorting); } private static int compareWithFallback(Hit left, Hit right, Sorting sorting, Comparator<Hit> fallback) { - if (left == null || right == null || !(left instanceof FastHit) || !(right instanceof FastHit)) { + if (!(left instanceof FastHit fl) || !(right instanceof FastHit fr)) { return fallback.compare(left, right); } - FastHit fl = (FastHit) left; - FastHit fr = (FastHit) right; if (fl.hasSortData(sorting) && fr.hasSortData(sorting)) { return FastHit.compareSortData(fl, fr, sorting); } else { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 7f421832d5f..8c154072a42 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; - import com.yahoo.collections.CopyOnWriteHashMap; import com.yahoo.compress.IntegerCompressor; import com.yahoo.language.Language; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java b/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java index 29cf7803d61..79fbeb99119 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.query; import java.nio.ByteBuffer; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -70,43 +71,26 @@ public class RegExpItem extends TermItem { putString(getIndexedString(), buffer); } + public Pattern getRegexp() { return regexp; } + @Override public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("RegExpItem [expression=").append(expression).append("]"); - return builder.toString(); + return "RegExpItem [expression=" + expression + "]"; } @Override public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((expression == null) ? 0 : expression.hashCode()); - return result; + return Objects.hash(super.hashCode(), expression); } @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!super.equals(obj)) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - RegExpItem other = (RegExpItem) obj; - if (expression == null) { - if (other.expression != null) { - return false; - } - } else if (!expression.equals(other.expression)) { - return false; - } - return true; - } + public boolean equals(Object o) { + if (this == o) return true; + if ( ! super.equals(o)) return false; + if (getClass() != o.getClass()) return false; - public Pattern getRegexp() { return regexp; } + RegExpItem other = (RegExpItem)o; + return Objects.equals(this.expression, other.expression); + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java index 6ecf9cd906f..47125d198e1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java @@ -20,7 +20,6 @@ import java.util.Map; import static com.yahoo.prelude.querytransform.NormalizingSearcher.ACCENT_REMOVAL; - /** * Check sorting specification makes sense to the search cluster before * passing it on to the backend. @@ -35,6 +34,13 @@ public class ValidateSortingSearcher extends Searcher { private String clusterName = ""; private final QrSearchersConfig.Searchcluster.Indexingmode.Enum indexingMode; + public ValidateSortingSearcher(QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, + AttributesConfig attributesConfig) { + initAttributeNames(attributesConfig); + setClusterName(qrsConfig.searchcluster(clusterConfig.clusterId()).name()); + indexingMode = qrsConfig.searchcluster(clusterConfig.clusterId()).indexingmode(); + } + public String getClusterName() { return clusterName; } @@ -63,14 +69,6 @@ public class ValidateSortingSearcher extends Searcher { setAttributeNames(attributes); } - public ValidateSortingSearcher(QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, - AttributesConfig attributesConfig) - { - initAttributeNames(attributesConfig); - setClusterName(qrsConfig.searchcluster(clusterConfig.clusterId()).name()); - indexingMode = qrsConfig.searchcluster(clusterConfig.clusterId()).indexingmode(); - } - @Override public Result search(Query query, Execution execution) { if (indexingMode != QrSearchersConfig.Searchcluster.Indexingmode.STREAMING) { @@ -157,8 +155,7 @@ public class ValidateSortingSearcher extends Searcher { } } } - if (f.getSorter() instanceof Sorting.UcaSorter) { - Sorting.UcaSorter sorter = (Sorting.UcaSorter) f.getSorter(); + if (f.getSorter() instanceof Sorting.UcaSorter sorter) { String locale = sorter.getLocale(); if (locale == null || locale.isEmpty()) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index b4f04da5986..39be91cf3e8 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -210,17 +210,17 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM indexPartial++; } } - while ((indexCurrent < current.size()) && (merged.size() < needed)) { - LeanHit currentHit = current.get(indexCurrent++); - merged.add(currentHit); - } - while ((indexPartial < partial.size()) && (merged.size() < needed)) { - LeanHit incomingHit = partial.get(indexPartial++); - merged.add(incomingHit); - } + appendRemainingIfNeeded(merged, needed, current, indexCurrent); + appendRemainingIfNeeded(merged, needed, partial, indexPartial); return merged; } + private void appendRemainingIfNeeded(List<LeanHit> merged, int needed, List<LeanHit> hits, int index) { + while ((index < hits.size()) && (merged.size() < needed)) { + merged.add(hits.get(index++)); + } + } + private void ejectInvoker(SearchInvoker invoker) { invokers.remove(invoker); invoker.release(); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index bd0415fa449..f80cebe90c4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -51,7 +51,7 @@ public class LeanHit implements Comparable<LeanHit> { return (res != 0) ? res : compareData(gid, o.gid); } - private static int compareData(byte[] left, byte[] right) { + public static int compareData(byte[] left, byte[] right) { int i = Arrays.mismatch(left, right); if (i < 0) { return 0; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MapConverter.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MapConverter.java index a71ec5d8345..7e54afcc070 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MapConverter.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MapConverter.java @@ -4,10 +4,11 @@ package com.yahoo.search.dispatch.rpc; import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol.StringProperty; import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol.TensorProperty; import com.google.protobuf.ByteString; +import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -17,12 +18,13 @@ import java.util.function.Consumer; */ public class MapConverter { - public static void convertMapTensors(Map<String, Object> map, Consumer<TensorProperty.Builder> inserter) { + public static void convertMapTensors(GrowableByteBuffer buffer, Map<String, Object> map, Consumer<TensorProperty.Builder> inserter) { for (var entry : map.entrySet()) { var value = entry.getValue(); - if (value instanceof Tensor) { - byte[] tensor = TypedBinaryFormat.encode((Tensor) value); - inserter.accept(TensorProperty.newBuilder().setName(entry.getKey()).setValue(ByteString.copyFrom(tensor))); + if (value instanceof Tensor tensor) { + buffer.clear(); + TypedBinaryFormat.encode(tensor, buffer); + inserter.accept(TensorProperty.newBuilder().setName(entry.getKey()).setValue(ByteString.copyFrom(buffer.getByteBuffer().flip()))); } } } @@ -45,18 +47,20 @@ public class MapConverter { } } - public static void convertMultiMap(Map<String, List<Object>> map, + public static void convertMultiMap(GrowableByteBuffer buffer, + Map<String, List<Object>> map, Consumer<StringProperty.Builder> stringInserter, Consumer<TensorProperty.Builder> tensorInserter) { for (var entry : map.entrySet()) { if (entry.getValue() != null) { var key = entry.getKey(); - var stringValues = new LinkedList<String>(); + var stringValues = new ArrayList<String>(entry.getValue().size()); for (var value : entry.getValue()) { if (value != null) { - if (value instanceof Tensor) { - byte[] tensor = TypedBinaryFormat.encode((Tensor) value); - tensorInserter.accept(TensorProperty.newBuilder().setName(key).setValue(ByteString.copyFrom(tensor))); + if (value instanceof Tensor tensor) { + buffer.clear(); + TypedBinaryFormat.encode(tensor, buffer); + tensorInserter.accept(TensorProperty.newBuilder().setName(key).setValue(ByteString.copyFrom(buffer.getByteBuffer().flip()))); } else { stringValues.add(value.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index de4f4f45eed..90e16e9dd44 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -38,7 +38,14 @@ import java.util.function.Consumer; public class ProtobufSerialization { - private static final int INITIAL_SERIALIZATION_BUFFER_SIZE = 10 * 1024; + /* + * This is a thread local buffer that is used as scratchpad during serialization. + * - avoids the unnecessary cost of allocating and initializing a buffer that is too large. + * - avoids resizing for large queries. + * - Reduces garbage creation. + * There is a limited number of threads that will use this so the upper bound should be fine. + */ + private static final ThreadLocal<GrowableByteBuffer> threadLocalBuffer = ThreadLocal.withInitial(() -> new GrowableByteBuffer(4096)); static byte[] serializeSearchRequest(Query query, int hits, String serverId, double requestTimeout) { return convertFromQuery(query, hits, serverId, requestTimeout).toByteArray(); @@ -58,7 +65,8 @@ public class ProtobufSerialization { if (documentDb != null) { builder.setDocumentType(documentDb); } - builder.setQueryTreeBlob(serializeQueryTree(query.getModel().getQueryTree())); + GrowableByteBuffer scratchPad = threadLocalBuffer.get(); + builder.setQueryTreeBlob(serializeQueryTree(query.getModel().getQueryTree(), scratchPad)); if (query.getGroupingSessionCache() || query.getRanking().getQueryCache()) { // TODO verify that the session key is included whenever rank properties would have been @@ -69,7 +77,8 @@ public class ProtobufSerialization { } if (GroupingExecutor.hasGroupingList(query)) { List<Grouping> groupingList = GroupingExecutor.getGroupingList(query); - BufferSerializer gbuf = new BufferSerializer(new GrowableByteBuffer()); + scratchPad.clear(); + BufferSerializer gbuf = new BufferSerializer(scratchPad); gbuf.putInt(null, groupingList.size()); for (Grouping g : groupingList) { g.serialize(gbuf); @@ -84,7 +93,7 @@ public class ProtobufSerialization { builder.setTraceLevel(getTraceLevelForBackend(query)); builder.setProfileDepth(query.getTrace().getProfileDepth()); - mergeToSearchRequestFromRanking(query.getRanking(), builder); + mergeToSearchRequestFromRanking(query.getRanking(), scratchPad, builder); return builder.build(); } @@ -100,7 +109,7 @@ public class ProtobufSerialization { return traceLevel; } - private static void mergeToSearchRequestFromRanking(Ranking ranking, SearchProtocol.SearchRequest.Builder builder) { + private static void mergeToSearchRequestFromRanking(Ranking ranking, GrowableByteBuffer scratchPad, SearchProtocol.SearchRequest.Builder builder) { builder.setRankProfile(ranking.getProfile()); if (ranking.getQueryCache()) { @@ -115,8 +124,8 @@ public class ProtobufSerialization { var featureMap = ranking.getFeatures().asMap(); MapConverter.convertMapPrimitives(featureMap, builder::addFeatureOverrides); - MapConverter.convertMapTensors(featureMap, builder::addTensorFeatureOverrides); - mergeRankProperties(ranking, builder::addRankProperties, builder::addTensorRankProperties); + MapConverter.convertMapTensors(scratchPad, featureMap, builder::addTensorFeatureOverrides); + mergeRankProperties(ranking, scratchPad, builder::addRankProperties, builder::addTensorRankProperties); } private static void mergeToSearchRequestFromSorting(Sorting sorting, SearchProtocol.SearchRequest.Builder builder) { @@ -160,8 +169,9 @@ public class ProtobufSerialization { if (ranking.getLocation() != null) { builder.setGeoLocation(ranking.getLocation().backendString()); } + GrowableByteBuffer scratchPad = threadLocalBuffer.get(); if (includeQueryData) { - mergeQueryDataToDocsumRequest(query, builder); + mergeQueryDataToDocsumRequest(query, scratchPad, builder); } if (query.getTrace().getLevel() >= 3) { query.trace((includeQueryData ? "ProtoBuf: Resending " : "Not resending ") + "query during document summary fetching", 3); @@ -178,18 +188,18 @@ public class ProtobufSerialization { return builder.build().toByteArray(); } - private static void mergeQueryDataToDocsumRequest(Query query, SearchProtocol.DocsumRequest.Builder builder) { + private static void mergeQueryDataToDocsumRequest(Query query, GrowableByteBuffer scratchPad, SearchProtocol.DocsumRequest.Builder builder) { var ranking = query.getRanking(); var featureMap = ranking.getFeatures().asMap(); - builder.setQueryTreeBlob(serializeQueryTree(query.getModel().getQueryTree())); + builder.setQueryTreeBlob(serializeQueryTree(query.getModel().getQueryTree(), scratchPad)); MapConverter.convertMapPrimitives(featureMap, builder::addFeatureOverrides); - MapConverter.convertMapTensors(featureMap, builder::addTensorFeatureOverrides); + MapConverter.convertMapTensors(scratchPad, featureMap, builder::addTensorFeatureOverrides); if (query.getPresentation().getHighlight() != null) { MapConverter.convertStringMultiMap(query.getPresentation().getHighlight().getHighlightTerms(), builder::addHighlightTerms); } - mergeRankProperties(ranking, builder::addRankProperties, builder::addTensorRankProperties); + mergeRankProperties(ranking, scratchPad, builder::addRankProperties, builder::addTensorRankProperties); } static byte[] serializeResult(Result searchResult) { return convertFromResult(searchResult).toByteArray(); @@ -297,24 +307,25 @@ public class ProtobufSerialization { return builder.build(); } - private static ByteString serializeQueryTree(QueryTree queryTree) { - int bufferSize = INITIAL_SERIALIZATION_BUFFER_SIZE; + private static ByteString serializeQueryTree(QueryTree queryTree, GrowableByteBuffer scratchPad) { while (true) { try { - ByteBuffer treeBuffer = ByteBuffer.allocate(bufferSize); + scratchPad.clear(); + ByteBuffer treeBuffer = scratchPad.getByteBuffer(); queryTree.encode(treeBuffer); - treeBuffer.flip(); - return ByteString.copyFrom(treeBuffer); + return ByteString.copyFrom(treeBuffer.flip()); } catch (java.nio.BufferOverflowException e) { - bufferSize *= 2; + scratchPad.clear(); + scratchPad.grow(scratchPad.capacity()*2); } } } private static void mergeRankProperties(Ranking ranking, + GrowableByteBuffer scratchPad, Consumer<StringProperty.Builder> stringProperties, Consumer<TensorProperty.Builder> tensorProperties) { - MapConverter.convertMultiMap(ranking.getProperties().asMap(), propB -> { + MapConverter.convertMultiMap(scratchPad, ranking.getProperties().asMap(), propB -> { if (!GetDocSumsPacket.sessionIdKey.equals(propB.getName())) { stringProperties.accept(propB); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java index bf0272f4f66..7dd1772a53e 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java +++ b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java @@ -22,9 +22,11 @@ import java.util.Map; * @author baldersheim */ class Json2SingleLevelMap { + private static final ObjectMapper jsonMapper = new ObjectMapper().configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); private final byte [] buf; private final JsonParser parser; + Json2SingleLevelMap(InputStream data) { try { buf = data.readAllBytes(); @@ -33,6 +35,7 @@ class Json2SingleLevelMap { throw new RuntimeException("Problem reading POSTed data", e); } } + Map<String, String> parse() { try { Map<String, String> map = new HashMap<>(); @@ -47,16 +50,17 @@ class Json2SingleLevelMap { throw new RuntimeException("Problem reading POSTed data", e); } } + void parse(Map<String, String> map, String parent) throws IOException { for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { String fieldName = parent + parser.getCurrentName(); JsonToken token = parser.nextToken(); if ((token == JsonToken.VALUE_STRING) || - (token == JsonToken.VALUE_NUMBER_FLOAT) || - (token == JsonToken.VALUE_NUMBER_INT) || - (token == JsonToken.VALUE_TRUE) || - (token == JsonToken.VALUE_FALSE) || - (token == JsonToken.VALUE_NULL)) { + (token == JsonToken.VALUE_NUMBER_FLOAT) || + (token == JsonToken.VALUE_NUMBER_INT) || + (token == JsonToken.VALUE_TRUE) || + (token == JsonToken.VALUE_FALSE) || + (token == JsonToken.VALUE_NULL)) { map.put(fieldName, parser.getText()); } else if (token == JsonToken.START_ARRAY) { map.put(fieldName, skipChildren(parser, buf)); @@ -71,6 +75,7 @@ class Json2SingleLevelMap { } } } + private String skipChildren(JsonParser parser, byte [] input) throws IOException { JsonLocation start = parser.getCurrentLocation(); parser.skipChildren(); @@ -78,4 +83,5 @@ class Json2SingleLevelMap { int offset = (int)start.getByteOffset() - 1; return new String(input, offset, (int)(end.getByteOffset() - offset), StandardCharsets.UTF_8); } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java index 6326097d9bd..1cc2b98c65b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java +++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java @@ -140,22 +140,18 @@ public class QueryTree extends CompositeItem { else if (b == null || b instanceof NullItem) { return a; } - else if (a instanceof NotItem && b instanceof NotItem) { - NotItem notItemA = (NotItem)a; - NotItem notItemB = (NotItem)b; + else if (a instanceof NotItem notItemA && b instanceof NotItem notItemB) { NotItem combined = new NotItem(); combined.addPositiveItem(and(notItemA.getPositiveItem(), notItemB.getPositiveItem())); notItemA.negativeItems().forEach(item -> combined.addNegativeItem(item)); notItemB.negativeItems().forEach(item -> combined.addNegativeItem(item)); return combined; } - else if (a instanceof NotItem){ - NotItem notItem = (NotItem)a; + else if (a instanceof NotItem notItem){ notItem.addPositiveItem(b); return a; } - else if (b instanceof NotItem){ - NotItem notItem = (NotItem)b; + else if (b instanceof NotItem notItem){ notItem.addPositiveItem(a); return notItem; } @@ -179,17 +175,16 @@ public class QueryTree extends CompositeItem { } private static void getPositiveTerms(Item item, List<IndexedItem> terms) { - if (item instanceof NotItem) { - getPositiveTerms(((NotItem) item).getPositiveItem(), terms); - } else if (item instanceof PhraseItem) { - PhraseItem pItem = (PhraseItem)item; - terms.add(pItem); - } else if (item instanceof CompositeItem) { - for (Iterator<Item> i = ((CompositeItem) item).getItemIterator(); i.hasNext();) { + if (item instanceof NotItem notItem) { + getPositiveTerms(notItem.getPositiveItem(), terms); + } else if (item instanceof PhraseItem phraseItem) { + terms.add(phraseItem); + } else if (item instanceof CompositeItem compositeItem) { + for (Iterator<Item> i = compositeItem.getItemIterator(); i.hasNext();) { getPositiveTerms(i.next(), terms); } - } else if (item instanceof TermItem) { - terms.add((TermItem)item); + } else if (item instanceof TermItem termItem) { + terms.add(termItem); } } @@ -203,8 +198,7 @@ public class QueryTree extends CompositeItem { private int countItemsRecursively(Item item) { int children = 0; - if (item instanceof CompositeItem) { - CompositeItem composite = (CompositeItem)item; + if (item instanceof CompositeItem composite) { for (ListIterator<Item> i = composite.getItemIterator(); i.hasNext(); ) { children += countItemsRecursively(i.next()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java index 6b07b5ef1d5..63bc48ff804 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Sorting.java +++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java @@ -6,7 +6,6 @@ import com.ibm.icu.util.ULocale; import com.yahoo.prelude.IndexFacts; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; -import com.yahoo.search.searchchain.Execution; import com.yahoo.text.Utf8; import java.nio.ByteBuffer; @@ -161,6 +160,7 @@ public class Sorting implements Cloneable { */ public List<FieldOrder> fieldOrders() { return fieldOrders; } + @Override public Sorting clone() { return new Sorting(this.fieldOrders); } @@ -173,16 +173,13 @@ public class Sorting implements Cloneable { @Override public boolean equals(Object o) { if (o == this) return true; - if( ! (o instanceof Sorting)) return false; - - Sorting ss = (Sorting) o; + if( ! (o instanceof Sorting ss)) return false; return fieldOrders.equals(ss.fieldOrders); } public int encode(ByteBuffer buffer) { int usedBytes = 0; byte[] nameBuffer; - buffer.position(); byte space = '.'; for (FieldOrder fieldOrder : fieldOrders) { if (space == ' ') { @@ -231,10 +228,10 @@ public class Sorting implements Cloneable { @Override public boolean equals(Object other) { - if (!(other instanceof AttributeSorter)) { + if (!(other instanceof AttributeSorter sorter)) { return false; } - return ((AttributeSorter) other).fieldName.equals(fieldName); + return sorter.fieldName.equals(fieldName); } @Override @@ -305,15 +302,14 @@ public class Sorting implements Cloneable { public UcaSorter(String fieldName) { super(fieldName); } static private int strength2Collator(Strength strength) { - switch (strength) { - case PRIMARY: return Collator.PRIMARY; - case SECONDARY: return Collator.SECONDARY; - case TERTIARY: return Collator.TERTIARY; - case QUATERNARY: return Collator.QUATERNARY; - case IDENTICAL: return Collator.IDENTICAL; - case UNDEFINED: return Collator.PRIMARY; - } - return Collator.PRIMARY; + return switch (strength) { + case PRIMARY -> Collator.PRIMARY; + case SECONDARY -> Collator.SECONDARY; + case TERTIARY -> Collator.TERTIARY; + case QUATERNARY -> Collator.QUATERNARY; + case IDENTICAL -> Collator.IDENTICAL; + case UNDEFINED -> Collator.PRIMARY; + }; } public void setLocale(String locale, Strength strength) { @@ -323,15 +319,15 @@ public class Sorting implements Cloneable { try { uloc = new ULocale(locale); } catch (Throwable e) { - throw new RuntimeException("ULocale("+locale+") failed with exception " + e.toString()); + throw new IllegalArgumentException("ULocale '" + locale + "' failed", e); } try { collator = Collator.getInstance(uloc); if (collator == null) { - throw new RuntimeException("No collator available for: " + locale); + throw new IllegalArgumentException("No collator available for locale '" + locale + "'"); } } catch (Throwable e) { - throw new RuntimeException("Collator.getInstance(ULocale("+locale+")) failed with exception " + e.toString()); + throw new RuntimeException("Collator.getInstance(ULocale(" + locale + ")) failed", e); } collator.setStrength(strength2Collator(strength)); // collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION); @@ -343,19 +339,22 @@ public class Sorting implements Cloneable { public String getDecomposition() { return (collator.getDecomposition() == Collator.CANONICAL_DECOMPOSITION) ? "CANONICAL_DECOMPOSITION" : "NO_DECOMPOSITION"; } @Override - public String toSerialForm() { return "uca(" + getName() + ',' + locale + ',' + ((strength != Strength.UNDEFINED) ? strength.toString() : "PRIMARY") + ')'; } + public String toSerialForm() { + return "uca(" + getName() + ',' + locale + ',' + + ((strength != Strength.UNDEFINED) ? strength.toString() : "PRIMARY") + ')'; + } @Override public int hashCode() { return 1 + 3*locale.hashCode() + 5*strength.hashCode() + 7*super.hashCode(); } @Override public boolean equals(Object other) { - if (!(other instanceof UcaSorter)) { - return false; - } + if (this == other) return true; + if (!(other instanceof UcaSorter)) return false; return super.equals(other) && locale.equals(((UcaSorter)other).locale) && (strength == ((UcaSorter)other).strength); } + @Override public UcaSorter clone() { UcaSorter clone = (UcaSorter)super.clone(); if (locale != null) { @@ -365,6 +364,7 @@ public class Sorting implements Cloneable { } @SuppressWarnings({ "rawtypes", "unchecked" }) + @Override public int compare(Comparable a, Comparable b) { if ((a instanceof String) && (b instanceof String)) { return collator.compare((String)a, (String) b); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 224f75e7034..f5d797b4b9f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -357,7 +357,7 @@ public class QueryProfileProperties extends Properties { if (profile.getTypes().isEmpty()) return name; CompoundName unaliasedName = name; - for (int i = 0; i<name.size(); i++) { + for (int i = 0; i < name.size(); i++) { QueryProfileType type = profile.getType(name.first(i), context); if (type == null) continue; if (type.aliases() == null) continue; // TODO: Make never null diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index afd25132510..bcdc84c1808 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -69,8 +69,7 @@ public class NGramSearcher extends Searcher { private boolean rewriteToNGramMatching(Item item, int indexInParent, IndexFacts.Session indexFacts, Query query) { boolean rewritten = false; - if (item instanceof SegmentItem) { // handle CJK segmented terms which should be grams instead - SegmentItem segments = (SegmentItem)item; + if (item instanceof SegmentItem segments) { // handle CJK segmented terms which should be grams instead Index index = indexFacts.getIndex(segments.getIndexName()); if (index.isNGram()) { Item grams = splitToGrams(segments, toLowerCase(segments.getRawWord()), index.getGramSize(), query); @@ -78,13 +77,11 @@ public class NGramSearcher extends Searcher { rewritten = true; } } - else if (item instanceof CompositeItem) { - CompositeItem composite = (CompositeItem)item; + else if (item instanceof CompositeItem composite) { for (int i=0; i<composite.getItemCount(); i++) rewritten = rewriteToNGramMatching(composite.getItem(i), i, indexFacts, query) || rewritten; } - else if (item instanceof TermItem) { - TermItem term = (TermItem)item; + else if (item instanceof TermItem term) { Index index = indexFacts.getIndex(term.getIndexName()); if (index.isNGram()) { Item grams = splitToGrams(term,term.stringValue(), index.getGramSize(), query); @@ -149,11 +146,10 @@ public class NGramSearcher extends Searcher { } private void replaceItemByGrams(Item item, Item grams, int indexInParent) { - if (!(grams instanceof CompositeItem) || !(item.getParent() instanceof PhraseItem)) { // usually, simply replace + if (!(grams instanceof CompositeItem) || !(item.getParent() instanceof PhraseItem phraseParent)) { // usually, simply replace item.getParent().setItem(indexInParent, grams); } else { // but if the parent is a phrase, we cannot add the AND to it, so add each gram to the phrase - PhraseItem phraseParent = (PhraseItem)item.getParent(); phraseParent.removeItem(indexInParent); int addedTerms = 0; for (Iterator<Item> i = ((CompositeItem)grams).getItemIterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java b/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java index 0259dd66dbe..7f392d43753 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java +++ b/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java @@ -17,7 +17,7 @@ import java.util.Comparator; public class FieldComparator extends ChainableComparator { /** The definition of sorting order */ - private Sorting sorting; + private final Sorting sorting; /** Creates a field comparator using a sort order and having no chained comparator */ public FieldComparator(Sorting sorting) { @@ -32,7 +32,7 @@ public class FieldComparator extends ChainableComparator { /** Creates a comparator given a sorting, or returns null if the given sorting is null */ public static FieldComparator create(Sorting sorting) { - if (sorting==null) return null; + if (sorting == null) return null; return new FieldComparator(sorting); } @@ -41,7 +41,7 @@ public class FieldComparator extends ChainableComparator { * stored in hit fields.0 * <p> * When one of the hits has the requested property and the other - * has not, the the hit containing the property precedes the one + * has not, the hit containing the property precedes the one * that does not. * <p> * There is no locale based sorting here, as the backend does @@ -78,19 +78,14 @@ public class FieldComparator extends ChainableComparator { } Inspector sub = top.field(key); if (sub.valid()) { - switch (sub.type()) { - case EMPTY: - return null; - case BOOL: - return (sub.asBool() ? Boolean.TRUE : Boolean.FALSE); - case LONG: - return sub.asLong(); - case DOUBLE: - return sub.asDouble(); - case STRING: - return sub.asString(); - } - return sub.toString(); + return switch (sub.type()) { + case EMPTY -> null; + case BOOL -> (sub.asBool() ? Boolean.TRUE : Boolean.FALSE); + case LONG -> sub.asLong(); + case DOUBLE -> sub.asDouble(); + case STRING -> sub.asString(); + default -> sub.toString(); + }; } } // fallback value @@ -115,15 +110,14 @@ public class FieldComparator extends ChainableComparator { @SuppressWarnings("rawtypes") private int compareValues(Object first, Object second, Sorting.AttributeSorter s) { - if (first == null) { - if (second == null) return 0; - return -1; - } else if (second == null) { + if (first == null) + return second == null ? 0 : -1; + else if (second == null) return 1; - } + if (first.getClass().isInstance(second) && first instanceof Comparable) { // We now know: - // second is of a type which is a subclass of first's type + // Second is of a type which is a subclass of first's type // They both implement Comparable return s.compare((Comparable)first, (Comparable)second); } else { @@ -133,14 +127,7 @@ public class FieldComparator extends ChainableComparator { @Override public String toString() { - StringBuilder b = new StringBuilder(); - b.append("FieldComparator:"); - if (sorting == null) { - b.append(" null"); - } else { - b.append(sorting.toString()); - } - return b.toString(); + return "FieldComparator:" + (sorting == null ? " null" : sorting.toString()); } } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index d7acccc75a7..0011d69fc2c 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -16,6 +16,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.function.BiConsumer; @@ -50,7 +51,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi private Map<String, Object> fields = null; private Map<String, Object> unmodifiableFieldMap = null; - /** Meta data describing how a given searcher should treat this hit. */ + /** Metadata describing how a given searcher should treat this hit. */ // TODO: The case for this is to allow multiple levels of federation searcher routing. // Replace this by a cleaner specific solution to that problem. private Map<Searcher, Object> searcherSpecificMetaData; @@ -289,7 +290,8 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * Returns the id to display, or null to not display (render) the id. * This is useful to avoid displaying ids when they are not assigned explicitly * but are just generated values for internal use. - * This default implementation returns {@link #getId()}.toString() + * This default implementation returns the field DOCUMENT_ID if set, + * and {@link #getId()}.toString() otherwise. */ public String getDisplayId() { String id = null; @@ -304,8 +306,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** Sets the relevance of this hit */ public void setRelevance(Relevance relevance) { - if (relevance == null) throw new NullPointerException("Cannot assign null as relevance"); - this.relevance = relevance; + this.relevance = Objects.requireNonNull(relevance); } /** Does setRelevance(new Relevance(relevance) */ @@ -330,7 +331,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * have been used for filling so far. Invoking this method * multiple times is allowed and will have no addition * effect. Note that a fillable hit may not be made unfillable. - **/ + */ public void setFillable() { if (filled == null) { filled = Collections.emptySet(); @@ -344,7 +345,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * tag this hit as fillable if it is currently not. * * @param summaryClass summary class used for filling - **/ + */ public void setFilled(String summaryClass) { if (filled == null || filled.isEmpty()) { filled = Collections.singleton(summaryClass); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java index adab4d59ec0..205c65a6256 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java @@ -67,7 +67,7 @@ public class AsyncExecution { * Creates an async execution. * * @param chain the chain to execute - * @param context the the context of this + * @param context the context of this */ public AsyncExecution(Chain<? extends Searcher> chain, Execution.Context context) { this(context, chain); diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index 5258087eb44..32880f9b1a8 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -277,29 +277,28 @@ final class ProgramParser { switch (getParseTreeIndex(sourceNode)) { // ALL_SOURCE and MULTI_SOURCE are how FROM SOURCES // *|source_name,... are parsed - case yqlplusParser.RULE_select_source_all: - Location location = toLocation(scope, sourceNode.getChild(2)); + case yqlplusParser.RULE_select_source_all -> { + Location location = toLocation(scope, sourceNode.getChild(2)); source = OperatorNode.create(location, SequenceOperator.ALL); source.putAnnotation("alias", "row"); scope.defineDataSource(location, "row"); - break; - case yqlplusParser.RULE_select_source_multi: - Source_listContext multiSourceContext = ((Select_source_multiContext) sourceNode).source_list(); + } + case yqlplusParser.RULE_select_source_multi -> { + Source_listContext multiSourceContext = ((Select_source_multiContext) sourceNode).source_list(); source = readMultiSource(scope, multiSourceContext); source.putAnnotation("alias", "row"); scope.defineDataSource(toLocation(scope, multiSourceContext), "row"); - break; - case yqlplusParser.RULE_select_source_from: - source = convertSource((ParserRuleContext) sourceNode.getChild(1), scope); - break; } - } else { - source = OperatorNode.create(SequenceOperator.EMPTY); + case yqlplusParser.RULE_select_source_from -> + source = convertSource((ParserRuleContext) sourceNode.getChild(1), scope); } + } else { + source = OperatorNode.create(SequenceOperator.EMPTY); + } - for (int i = 1; i < node.getChildCount(); ++i) { - ParseTree child = node.getChild(i); - switch (getParseTreeIndex(child)) { + for (int i = 1; i < node.getChildCount(); ++i) { + ParseTree child = node.getChild(i); + switch (getParseTreeIndex(child)) { case yqlplusParser.RULE_select_field_spec: if (getParseTreeIndex(child.getChild(0)) == yqlplusParser.RULE_project_spec) { proj = readProjection(((Project_specContext) child.getChild(0)).field_def(), scope); @@ -311,7 +310,7 @@ final class ProgramParser { case yqlplusParser.RULE_orderby: // OrderbyContext orderby() List<Orderby_fieldContext> orderFieds = ((OrderbyContext) child) - .orderby_fields().orderby_field(); + .orderby_fields().orderby_field(); orderby = Lists.newArrayListWithExpectedSize(orderFieds.size()); for (var field: orderFieds) { orderby.add(convertSortKey(field, scope)); @@ -326,8 +325,8 @@ final class ProgramParser { case yqlplusParser.RULE_timeout: timeout = convertExpr(((TimeoutContext) child).fixed_or_parameter(), scope); break; - } } + } // now assemble the logical plan OperatorNode<SequenceOperator> result = source; // filter @@ -415,8 +414,7 @@ final class ProgramParser { private OperatorNode<SequenceOperator> convertQuery(ParseTree node, Scope scope) { if (node instanceof Select_statementContext) { return convertSelect(node, scope.getRoot()); - } else if (node instanceof Source_statementContext) { // for pipe - Source_statementContext sourceStatementContext = (Source_statementContext)node; + } else if (node instanceof Source_statementContext sourceStatementContext) { // for pipe return convertPipe(sourceStatementContext.query_statement(), sourceStatementContext.pipeline_step(), scope); } else { throw new IllegalArgumentException("Unexpected argument type to convertQueryStatement: " + node.toStringTree()); @@ -469,47 +467,44 @@ final class ProgramParser { dataSourceNode = (ParserRuleContext)dataSourceNode.getChild(1); } } - switch (getParseTreeIndex(dataSourceNode)) { - case yqlplusParser.RULE_call_source: { - List<String> names = readName(dataSourceNode.getChild(Namespaced_nameContext.class, 0)); - alias = assignAlias(names.get(names.size() - 1), aliasContext, scope); - List<OperatorNode<ExpressionOperator>> arguments = ImmutableList.of(); - ArgumentsContext argumentsContext = dataSourceNode.getRuleContext(ArgumentsContext.class,0); - if ( argumentsContext != null) { - List<ArgumentContext> argumentContexts = argumentsContext.argument(); - arguments = Lists.newArrayListWithExpectedSize(argumentContexts.size()); - for (ArgumentContext argumentContext:argumentContexts) { - arguments.add(convertExpr(argumentContext, scope)); - } - } - if (names.size() == 1 && scope.isVariable(names.get(0))) { - String ident = names.get(0); - if (arguments.size() > 0) { - throw new ProgramCompileException(toLocation(scope, argumentsContext), "Invalid call-with-arguments on local source '%s'", ident); - } - result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.EVALUATE, OperatorNode.create(toLocation(scope, dataSourceNode), ExpressionOperator.VARREF, ident)); + switch (getParseTreeIndex(dataSourceNode)) { + case yqlplusParser.RULE_call_source -> { + List<String> names = readName(dataSourceNode.getChild(Namespaced_nameContext.class, 0)); + alias = assignAlias(names.get(names.size() - 1), aliasContext, scope); + List<OperatorNode<ExpressionOperator>> arguments = ImmutableList.of(); + ArgumentsContext argumentsContext = dataSourceNode.getRuleContext(ArgumentsContext.class, 0); + if (argumentsContext != null) { + List<ArgumentContext> argumentContexts = argumentsContext.argument(); + arguments = Lists.newArrayListWithExpectedSize(argumentContexts.size()); + for (ArgumentContext argumentContext : argumentContexts) { + arguments.add(convertExpr(argumentContext, scope)); + } + } + if (names.size() == 1 && scope.isVariable(names.get(0))) { + String ident = names.get(0); + if (arguments.size() > 0) { + throw new ProgramCompileException(toLocation(scope, argumentsContext), "Invalid call-with-arguments on local source '%s'", ident); + } + result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.EVALUATE, OperatorNode.create(toLocation(scope, dataSourceNode), ExpressionOperator.VARREF, ident)); } else { - result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.SCAN, scope.resolvePath(names), arguments); + result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.SCAN, scope.resolvePath(names), arguments); } - break; } - case yqlplusParser.RULE_sequence_source: { - IdentContext identContext = dataSourceNode.getRuleContext(IdentContext.class,0); + case yqlplusParser.RULE_sequence_source -> { + IdentContext identContext = dataSourceNode.getRuleContext(IdentContext.class, 0); String ident = identContext.getText(); if (!scope.isVariable(ident)) { throw new ProgramCompileException(toLocation(scope, identContext), "Unknown variable reference '%s'", ident); } alias = assignAlias(ident, aliasContext, scope); result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.EVALUATE, OperatorNode.create(toLocation(scope, dataSourceNode), ExpressionOperator.VARREF, ident)); - break; } - case yqlplusParser.RULE_source_statement: { + case yqlplusParser.RULE_source_statement -> { alias = assignAlias(null, dataSourceNode, scope); result = convertQuery(dataSourceNode, scope); - break; } - default: - throw new IllegalArgumentException("Unexpected argument type to convertSource: " + dataSourceNode.getText()); + default -> + throw new IllegalArgumentException("Unexpected argument type to convertSource: " + dataSourceNode.getText()); } result.putAnnotation("alias", alias); return result; @@ -522,10 +517,7 @@ final class ProgramParser { List<OperatorNode<StatementOperator>> stmts = Lists.newArrayList(); int output = 0; for (ParseTree node : program.children) { - if (!(node instanceof ParserRuleContext)) { - continue; - } - ParserRuleContext ruleContext = (ParserRuleContext) node; + if (!(node instanceof ParserRuleContext ruleContext)) continue; if (ruleContext.getRuleIndex() != yqlplusParser.RULE_statement) throw new ProgramCompileException("Unknown program element: " + node.getText()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index cab989d4466..4b035e0ccc5 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -121,7 +121,6 @@ import com.yahoo.search.query.QueryTree; public class VespaSerializer { // TODO: Refactor, too much copy/paste - private static abstract class Serializer<ITEM extends Item> { abstract void onExit(StringBuilder destination, ITEM item); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 72cb102760a..3a1927983d2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -90,11 +90,9 @@ import com.yahoo.search.query.parser.ParserFactory; /** * The YQL query language. * - * <p> * This class <em>must</em> be kept in lockstep with {@link VespaSerializer}. * Adding anything here will usually require a corresponding addition in * VespaSerializer. - * </p> * * @author Steinar Knutsen * @author Stian Kristoffersen @@ -132,8 +130,9 @@ public class YqlParser implements Parser { private static final String USER_INPUT_DEFAULT_INDEX = "defaultIndex"; private static final String USER_INPUT_GRAMMAR = "grammar"; public static final String USER_INPUT_LANGUAGE = "language"; - private static final String USER_INPUT_RAW = "raw"; - private static final String USER_INPUT_SEGMENT = "segment"; + private static final String USER_INPUT_GRAMMAR_RAW = "raw"; + private static final String USER_INPUT_GRAMMAR_SEGMENT = "segment"; + private static final String USER_INPUT_GRAMMAR_WEAKAND = "weakAnd"; private static final String USER_INPUT = "userInput"; private static final String USER_QUERY = "userQuery"; private static final String NON_EMPTY = "nonEmpty"; @@ -722,14 +721,19 @@ public class YqlParser implements Parser { String.class, "default", "default index for user input terms"); Language language = decideParsingLanguage(ast, wordData); Item item; - if (USER_INPUT_RAW.equals(grammar)) { + if (USER_INPUT_GRAMMAR_RAW.equals(grammar)) { item = instantiateWordItem(defaultIndex, wordData, ast, null, SegmentWhen.NEVER, true, language); - } else if (USER_INPUT_SEGMENT.equals(grammar)) { + } else if (USER_INPUT_GRAMMAR_SEGMENT.equals(grammar)) { item = instantiateWordItem(defaultIndex, wordData, ast, null, SegmentWhen.ALWAYS, false, language); } else { item = parseUserInput(grammar, defaultIndex, wordData, language, allowEmpty); propagateUserInputAnnotations(ast, item); } + + // Set grammar-specific annotations + if (USER_INPUT_GRAMMAR_WEAKAND.equals(grammar) && item instanceof WeakAndItem weakAndItem) { + weakAndItem.setN(getAnnotation(ast, TARGET_HITS, Integer.class, WeakAndItem.defaultN, "'targetHits' (N) for weak and")); + } return item; } @@ -1428,7 +1432,7 @@ public class YqlParser implements Parser { wordItem = new WordItem(wordData, fromQuery); break; case POSSIBLY: - if (shouldSegment(field, fromQuery) && ! grammar.equals(USER_INPUT_RAW)) { + if (shouldSegment(field, fromQuery) && ! grammar.equals(USER_INPUT_GRAMMAR_RAW)) { wordItem = segment(field, ast, wordData, fromQuery, parent, language); } else { wordItem = new WordItem(wordData, fromQuery); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index ae2a8800bbc..5b972e40774 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -65,7 +65,7 @@ import static org.junit.jupiter.api.Assertions.*; public class QueryTestCase { @Test - void testIt() throws Exception { + void testIt() { JSONObject newroot = new JSONObject("{\"key\": 3}"); var hit = new FastHit(); hit.setField("data", (JsonProducer) s -> s.append(newroot)); diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index 1e3b52c23af..858d5a16352 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -7,6 +7,7 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.query.WeakAndItem; import org.apache.http.client.utils.URIBuilder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -89,6 +90,17 @@ public class UserInputTestCase { } @Test + void testUserInputSettingTargetHits() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where {grammar: \"weakAnd\", targetHits: 17, defaultIndex: \"f\"}userInput(\"a test\")"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where ({targetNumHits: 17}weakAnd(f contains \"a\", f contains \"test\"))", query.yqlRepresentation()); + WeakAndItem weakAnd = (WeakAndItem)query.getModel().getQueryTree().getRoot(); + assertEquals(17, weakAnd.getN()); + } + + @Test void testSegmentedNoiseUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 5beea5352aa..807681e1d7b 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -1002,10 +1002,21 @@ public class YqlParserTestCase { @Test void testRegexp() { - QueryTree x = parse("select * from sources * where foo matches \"a b\""); - Item root = x.getRoot(); - assertSame(RegExpItem.class, root.getClass()); - assertEquals("a b", ((RegExpItem) root).stringValue()); + { + QueryTree x = parse("select * from sources * where foo matches \"a b\""); + Item root = x.getRoot(); + assertSame(RegExpItem.class, root.getClass()); + assertEquals("a b", ((RegExpItem) root).stringValue()); + } + + { + String expression = "a\\\\.b\\\\.c"; + QueryTree query = parse("select * from sources * where foo matches \"" + expression + "\""); + var regExpItem = (RegExpItem) query.getRoot(); + assertEquals("a\\.b\\.c", regExpItem.stringValue()); + assertTrue(regExpItem.getRegexp().matcher("a.b.c").matches(), "a.b.c is matched"); + assertFalse(regExpItem.getRegexp().matcher("a,b,c").matches(), "a,b,c is matched?"); + } } @Test |