diff options
author | Jon Bratseth <bratseth@vespa.ai> | 2023-06-20 22:27:54 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@vespa.ai> | 2023-06-20 22:27:54 +0200 |
commit | 687fc90a66d3bf96ec9af8a317b7ecdc00fd1eee (patch) | |
tree | 5effa22bdd651e32d6ec8b869f2376511b0c03fa /container-search | |
parent | bfcbaf4deea360bb073b7562b2e3c6bd26378f10 (diff) |
Non-functional changes only
Diffstat (limited to 'container-search')
10 files changed, 205 insertions, 221 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 46332d632fe..083de083108 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -24,7 +24,7 @@ import com.yahoo.search.ranking.GlobalPhaseRanker; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; +import com.yahoo.vespa.streamingvisitors.StreamingSearcher; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; @@ -98,8 +98,8 @@ public class ClusterSearcher extends Searcher { String uniqueServerId = UUID.randomUUID().toString(); if (searchClusterConfig.indexingmode() == STREAMING) { - server = vdsCluster(uniqueServerId, searchClusterIndex, - searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); + server = streamingCluster(uniqueServerId, searchClusterIndex, + searchClusterConfig, docSumParams, documentDbConfig, schemaInfo, access); vipStatus.addToRotation(server.getName()); } else { server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, @@ -136,18 +136,18 @@ public class ClusterSearcher extends Searcher { return new FastSearcher(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); } - private static VdsStreamingSearcher vdsCluster(String serverId, - int searchclusterIndex, - QrSearchersConfig.Searchcluster searchClusterConfig, - SummaryParameters docSumParams, - DocumentdbInfoConfig documentdbInfoConfig, - SchemaInfo schemaInfo, - VespaDocumentAccess access) { - if (searchClusterConfig.searchdef().size() != 1) { - throw new IllegalArgumentException("Search clusters in streaming search shall only contain a single schema : " + searchClusterConfig.searchdef()); - } + private static StreamingSearcher streamingCluster(String serverId, + int searchclusterIndex, + QrSearchersConfig.Searchcluster searchClusterConfig, + SummaryParameters docSumParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo, + VespaDocumentAccess access) { + if (searchClusterConfig.searchdef().size() != 1) + throw new IllegalArgumentException("Streaming search clusters can only contain a single schema but got " + + searchClusterConfig.searchdef()); ClusterParams clusterParams = makeClusterParams(searchclusterIndex); - VdsStreamingSearcher searcher = new VdsStreamingSearcher(access); + StreamingSearcher searcher = new StreamingSearcher(access); searcher.setSearchClusterName(searchClusterConfig.rankprofiles().configid()); searcher.setDocumentType(searchClusterConfig.searchdef(0)); searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec()); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java index 205c65a6256..8981a1c509f 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java @@ -102,7 +102,7 @@ public class AsyncExecution { } /** - * Does an async search, note that the query argument cannot simultaneously + * Does an async search. Note that the query argument cannot simultaneously * be used to execute any other searches, a clone() must be made of the * query for each async execution if the same query is to be used in more * than one. diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 988021a6da0..6227430ba42 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -471,7 +471,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { */ @SuppressWarnings("unchecked") private Execution(Chain<? extends Processor> searchChain, Context context, int searcherIndex) { - // Create a new Execution which is placed in the context of the execution of the given Context if any + // Create a new Execution which is placed in the context of the execution of the given Context if any. // "if any" because a context may, or may not, belong to an execution. // This is decided at the creation time of the Context - Context instances which do not belong // to an execution plays the role of data carriers between executions. diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index f1a49fc1231..a04e1459d68 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -44,7 +44,7 @@ public class InputCheckingSearcher extends Searcher { private final Counter repeatedTermsInPhraseRejections; private static final Logger log = Logger.getLogger(InputCheckingSearcher.class.getName()); private final int MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE = 5; - private final int MAX_REPEATED_TERMS_IN_PHRASE=10; + private final int MAX_REPEATED_TERMS_IN_PHRASE = 10; public InputCheckingSearcher(MetricReceiver metrics) { utfRejections = metrics.declareCounter("double_encoded_utf8_rejections"); @@ -70,12 +70,10 @@ public class InputCheckingSearcher extends Searcher { } private void checkPhrases(Item queryItem) { - if (queryItem instanceof PhraseItem) { - PhraseItem phrase = (PhraseItem) queryItem; + if (queryItem instanceof PhraseItem phrase) { repeatedConsecutiveTermsInPhraseCheck(phrase); repeatedTermsInPhraseCheck(phrase); - } else if (queryItem instanceof CompositeItem) { - CompositeItem asComposite = (CompositeItem) queryItem; + } else if (queryItem instanceof CompositeItem asComposite) { for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) { checkPhrases(i.next()); } @@ -88,8 +86,7 @@ public class InputCheckingSearcher extends Searcher { int repeatedCount = 0; for (int i = 0; i < phrase.getItemCount(); ++i) { Item item = phrase.getItem(i); - if (item instanceof TermItem) { - TermItem term = (TermItem) item; + if (item instanceof TermItem term) { String current = term.getIndexedString(); if (prev != null) { if (prev.equals(current)) { @@ -123,8 +120,7 @@ public class InputCheckingSearcher extends Searcher { Map<String, Count> repeatedCount = new HashMap<>(); for (int i = 0; i < phrase.getItemCount(); ++i) { Item item = phrase.getItem(i); - if (item instanceof TermItem) { - TermItem term = (TermItem) item; + if (item instanceof TermItem term) { String current = term.getIndexedString(); Count count = repeatedCount.get(current); if (count != null) { @@ -179,15 +175,13 @@ public class InputCheckingSearcher extends Searcher { } private int countSingleCharacterUserTerms(Item queryItem) { - if (queryItem instanceof CompositeItem) { + if (queryItem instanceof CompositeItem asComposite) { int sum = 0; - CompositeItem asComposite = (CompositeItem) queryItem; for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) { sum += countSingleCharacterUserTerms(i.next()); } return sum; - } else if (queryItem instanceof WordItem) { - WordItem word = (WordItem) queryItem; + } else if (queryItem instanceof WordItem word) { return (word.isFromQuery() && word.stringValue().length() == 1) ? 1 : 0; } else { return 0; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index 536355ab62d..762aec81c49 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -15,7 +15,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.logging.Logger; -import static com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher.STREAMING_STATISTICS; +import static com.yahoo.vespa.streamingvisitors.StreamingSearcher.STREAMING_STATISTICS; /** * Generates mail-specific query metrics. diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java index f8df6bbea03..87d07df7230 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingSearcher.java @@ -44,7 +44,7 @@ import java.util.logging.Logger; * @author baldersheim * @author Ulf Carlin */ -public class VdsStreamingSearcher extends VespaBackEndSearcher { +public class StreamingSearcher extends VespaBackEndSearcher { private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); @@ -53,60 +53,40 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { static final String STREAMING_STATISTICS = "streaming.statistics"; private final VisitorFactory visitorFactory; private final TracingOptions tracingOptions; - private static final Logger log = Logger.getLogger(VdsStreamingSearcher.class.getName()); + private static final Logger log = Logger.getLogger(StreamingSearcher.class.getName()); private Route route; + /** The configId used to access the searchcluster. */ private String searchClusterName = null; private String documentType; + /** The route to the storage cluster. */ private String storageClusterRouteSpec = null; - private String getSearchClusterName() { return searchClusterName; } - private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; } - public final void setSearchClusterName(String clusterName) { - this.searchClusterName = clusterName; - } - public final void setDocumentType(String documentType) { - this.documentType = documentType; - } - - public final void setStorageClusterRouteSpec(String storageClusterRouteSpec) { - this.storageClusterRouteSpec = storageClusterRouteSpec; + StreamingSearcher(VisitorFactory visitorFactory) { + this.visitorFactory = visitorFactory; + tracingOptions = TracingOptions.DEFAULT; } - private static class VespaVisitorFactory implements VdsVisitor.VisitorSessionFactory, VisitorFactory { - - private final VespaDocumentAccess access; - - private VespaVisitorFactory(VespaDocumentAccess access) { - this.access = access; - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { - return access.createVisitorSession(params); - } - - @Override - public Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride) { - return new VdsVisitor(query, searchCluster, route, documentType, this, traceLevelOverride); - } - + StreamingSearcher(VisitorFactory visitorFactory, TracingOptions tracingOptions) { + this.visitorFactory = visitorFactory; + this.tracingOptions = tracingOptions; } - public VdsStreamingSearcher(VespaDocumentAccess access) { + public StreamingSearcher(VespaDocumentAccess access) { this(new VespaVisitorFactory(access)); } - VdsStreamingSearcher(VisitorFactory visitorFactory) { - this.visitorFactory = visitorFactory; - tracingOptions = TracingOptions.DEFAULT; + private String getSearchClusterName() { return searchClusterName; } + private String getStorageClusterRouteSpec() { return storageClusterRouteSpec; } + public final void setSearchClusterName(String clusterName) { this.searchClusterName = clusterName; } + public final void setDocumentType(String documentType) { + this.documentType = documentType; } - VdsStreamingSearcher(VisitorFactory visitorFactory, TracingOptions tracingOptions) { - this.visitorFactory = visitorFactory; - this.tracingOptions = tracingOptions; + public final void setStorageClusterRouteSpec(String storageClusterRouteSpec) { + this.storageClusterRouteSpec = storageClusterRouteSpec; } @Override @@ -128,15 +108,12 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { private static int documentSelectionQueryParameterCount(Query query) { int paramCount = 0; - if (query.properties().getString(streamingUserid) != null) { + if (query.properties().getString(streamingUserid) != null) paramCount++; - } - if (query.properties().getString(streamingGroupname) != null) { + if (query.properties().getString(streamingGroupname) != null) paramCount++; - } - if (query.properties().getString(streamingSelection) != null) { + if (query.properties().getString(streamingSelection) != null) paramCount++; - } return paramCount; } @@ -154,17 +131,18 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { @Override public Result doSearch2(Query query, Execution execution) { - if (query.getTimeLeft() <= 0) { + if (query.getTimeLeft() <= 0) return new Result(query, ErrorMessage.createTimeout(String.format("No time left for searching (timeout=%d)", query.getTimeout()))); - } initializeMissingQueryFields(query); if (documentSelectionQueryParameterCount(query) != 1) { return new Result(query, ErrorMessage.createBackendCommunicationError("Streaming search needs one and " + - "only one of these query parameters to be set: streaming.userid, streaming.groupname, " + - "streaming.selection")); + "only one of these query parameters to be set: " + + "streaming.userid, streaming.groupname, or " + + "streaming.selection")); } - query.trace("Routing to search cluster " + getSearchClusterName() + " and document type " + documentType, 4); + if (query.getTrace().isTraceable(4)) + query.trace("Routing to search cluster " + getSearchClusterName() + " and document type " + documentType, 4); long timeStartedNanos = tracingOptions.getClock().nanoTimeNow(); int effectiveTraceLevel = inferEffectiveQueryTraceLevel(query); @@ -172,17 +150,17 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { try { visitor.doSearch(); } catch (ParseException e) { - return new Result(query, ErrorMessage.createBackendCommunicationError( - "Failed to parse document selection string: " + e.getMessage() + "'.")); + return new Result(query, ErrorMessage.createBackendCommunicationError("Failed to parse document selection string: " + + e.getMessage() + "'.")); } catch (TokenMgrException e) { - return new Result(query, ErrorMessage.createBackendCommunicationError( - "Failed to tokenize document selection string: " + e.getMessage() + "'.")); + return new Result(query, ErrorMessage.createBackendCommunicationError("Failed to tokenize document selection string: " + + e.getMessage() + "'.")); } catch (TimeoutException e) { double elapsedMillis = durationInMillisFromNanoTime(timeStartedNanos); if ((effectiveTraceLevel > 0) && timeoutBadEnoughToBeReported(query, elapsedMillis)) { tracingOptions.getTraceExporter().maybeExport(() -> new TraceDescription(visitor.getTrace(), - String.format("Trace of %s which timed out after %.3g seconds", - query.toString(), elapsedMillis / 1000.0))); + String.format("Trace of %s which timed out after %.3g seconds", + query, elapsedMillis / 1000.0))); } return new Result(query, ErrorMessage.createTimeout(e.getMessage())); } catch (InterruptedException | IllegalArgumentException e) { @@ -229,8 +207,8 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { Map<String, DocumentSummary.Summary> summaryMap = visitor.getSummaryMap(); lazyTrace(query, 7, "total hit count = ", visitor.getTotalHitCount(), - ", returned hit count = ", hits.size(), ", summary count = ", - summaryMap.size()); + ", returned hit count = ", hits.size(), ", summary count = ", + summaryMap.size()); VisitorStatistics stats = visitor.getStatistics(); result.setTotalHitCount(visitor.getTotalHitCount()); @@ -255,8 +233,8 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { DocsumPacket dp = new DocsumPacket(summary.getSummary()); summaryPackets[index] = dp; } else { - return new Result(query, ErrorMessage.createBackendCommunicationError( - "Did not find summary for hit with document id " + hit.getDocId())); + return new Result(query, ErrorMessage.createBackendCommunicationError("Did not find summary for hit with document id " + + hit.getDocId())); } index++; @@ -291,7 +269,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { if (skippedHits > 0) { getLogger().info("skipping " + skippedHits + " hits for query: " + result.getQuery()); - result.hits().addError(com.yahoo.search.result.ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits")); + result.hits().addError(ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits")); } return result; @@ -325,8 +303,8 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { } static boolean verifyDocId(String id, Query query, boolean skippedEarlierResult) { - String expUserId = query.properties().getString(streamingUserid); - String expGroupName = query.properties().getString(streamingGroupname); + String expectedUserId = query.properties().getString(streamingUserid); + String expectedGroupName = query.properties().getString(streamingGroupname); Level logLevel = Level.SEVERE; if (skippedEarlierResult) { @@ -341,7 +319,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { return false; } - if (expUserId != null) { + if (expectedUserId != null) { long userId; if (docId.getScheme().hasNumber()) { @@ -350,12 +328,12 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { log.log(logLevel, "Got result with wrong scheme in document ID (" + id + ") for " + query); return false; } - if (new BigInteger(expUserId).longValue() != userId) { - log.log(logLevel, "Got result with wrong user ID (expected " + expUserId + ") in document ID (" + - id + ") for " + query); + if (new BigInteger(expectedUserId).longValue() != userId) { + log.log(logLevel, "Got result with wrong user ID (expected " + expectedUserId + ") in document ID (" + + id + ") for " + query); return false; } - } else if (expGroupName != null) { + } else if (expectedGroupName != null) { String groupName; if (docId.getScheme().hasGroup()) { @@ -364,9 +342,9 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { log.log(logLevel, "Got result with wrong scheme in document ID (" + id + ") for " + query); return false; } - if (!expGroupName.equals(groupName)) { - log.log(logLevel, "Got result with wrong group name (expected " + expGroupName + ") in document ID (" + - id + ") for " + query); + if (!expectedGroupName.equals(groupName)) { + log.log(logLevel, "Got result with wrong group name (expected " + expectedGroupName + ") in document ID (" + + id + ") for " + query); return false; } } @@ -377,4 +355,25 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { // TODO add a real pong return new Pong(); } + + private static class VespaVisitorFactory implements StreamingVisitor.VisitorSessionFactory, VisitorFactory { + + private final VespaDocumentAccess access; + + private VespaVisitorFactory(VespaDocumentAccess access) { + this.access = access; + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { + return access.createVisitorSession(params); + } + + @Override + public Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride) { + return new StreamingVisitor(query, searchCluster, route, documentType, this, traceLevelOverride); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java index 409b807c833..85092bb23f0 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/StreamingVisitor.java @@ -39,7 +39,7 @@ import java.util.logging.Logger; import java.util.logging.Level; /** - * A visitor data handler that performs a query in VDS with the + * A visitor data handler that performs a query in a content cluster with the * searchvisitor visitor plugin. It collects and merges hits (sorted * descending on rank), summaries (sorted on document id), and * groupings. The resulting data can be fetched when the query has @@ -47,7 +47,7 @@ import java.util.logging.Level; * * @author Ulf Carlin */ -class VdsVisitor extends VisitorDataHandler implements Visitor { +class StreamingVisitor extends VisitorDataHandler implements Visitor { private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); @@ -59,7 +59,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { protected static final int MAX_BUCKETS_PER_VISITOR = 1024; - private static final Logger log = Logger.getLogger(VdsVisitor.class.getName()); + private static final Logger log = Logger.getLogger(StreamingVisitor.class.getName()); private final VisitorParameters params = new VisitorParameters(""); private List<SearchResult.Hit> hits = new ArrayList<>(); private int totalHitCount = 0; @@ -75,9 +75,9 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { VisitorSession createVisitorSession(VisitorParameters params) throws ParseException; } - public VdsVisitor(Query query, String searchCluster, Route route, - String documentType, VisitorSessionFactory visitorSessionFactory, - int traceLevelOverride) + public StreamingVisitor(Query query, String searchCluster, Route route, + String documentType, VisitorSessionFactory visitorSessionFactory, + int traceLevelOverride) { this.query = query; this.visitorSessionFactory = visitorSessionFactory; @@ -97,21 +97,18 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private static String createSelectionString(String documentType, String selection) { if ((selection == null) || selection.isEmpty()) return documentType; - - StringBuilder sb = new StringBuilder(documentType); - sb.append(" and ( ").append(selection).append(" )"); - return sb.toString(); + return documentType + " and ( " + selection + " )"; } private String createQuerySelectionString() { - String s = query.properties().getString(streamingUserid); - if (s != null) { - return "id.user==" + s; - } - s = query.properties().getString(streamingGroupname); - if (s != null) { - return "id.group==\"" + s + "\""; - } + String userId = query.properties().getString(streamingUserid); + if (userId != null) + return "id.user==" + userId; + + String groupId = query.properties().getString(streamingGroupname); + if (groupId != null) + return "id.group==\"" + groupId + "\""; + return query.properties().getString(streamingSelection); } @@ -140,7 +137,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.setMaxBucketsPerVisitor(MAX_BUCKETS_PER_VISITOR); params.setTraceLevel(inferSessionTraceLevel(query)); - String maxbuckets = query.properties().getString(streamingMaxbucketspervisitor); if (maxbuckets != null) { params.setMaxBucketsPerVisitor(Integer.parseInt(maxbuckets)); @@ -215,6 +211,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } private static class EncodedData { + private Object returned; private byte[] encoded; @@ -230,27 +227,23 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public byte[] getEncodedData(){ return encoded; } + } private static void encodeQueryData(Query query, int code, EncodedData ed){ ByteBuffer buf = ByteBuffer.allocate(1024); while (true) { try { - switch(code){ - case 0: - ed.setReturned(query.getModel().getQueryTree().getRoot().encode(buf)); - break; - case 1: - ed.setReturned(QueryEncoder.encodeAsProperties(query, buf)); - break; - case 2: - throw new IllegalArgumentException("old aggregation no longer exists!"); - case 3: + switch (code) { + case 0 -> ed.setReturned(query.getModel().getQueryTree().getRoot().encode(buf)); + case 1 -> ed.setReturned(QueryEncoder.encodeAsProperties(query, buf)); + case 2 -> throw new IllegalArgumentException("old aggregation no longer exists!"); + case 3 -> { if (query.getRanking().getSorting() != null) ed.setReturned(query.getRanking().getSorting().encode(buf)); else ed.setReturned(0); - break; + } } buf.flip(); break; @@ -269,9 +262,10 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { VisitorSession session = visitorSessionFactory.createVisitorSession(params); try { if ( ! session.waitUntilDone(query.getTimeout())) { - log.log(Level.FINE, () -> "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); + log.log(Level.FINE, () -> "StreamingVisitor returned from waitUntilDone without being completed for " + query + + " with selection " + params.getDocumentSelection()); session.abort(); - throw new TimeoutException("Query timed out in " + VdsStreamingSearcher.class.getName()); + throw new TimeoutException("Query timed out in " + StreamingSearcher.class.getName()); } } finally { session.destroy(); @@ -281,7 +275,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } if (params.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) { - log.log(Level.FINE, () -> "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); + log.log(Level.FINE, () -> "StreamingVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); } else { throw new IllegalArgumentException("Query failed: " + params.getControlHandler().getResult().code + ": " + @@ -299,7 +293,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { if (m instanceof QueryResultMessage qm) { onQueryResult(qm.getResult(), qm.getSummary()); } else { - throw new UnsupportedOperationException("Received unsupported message " + m + ". VdsVisitor can only accept query result messages."); + throw new UnsupportedOperationException("Received unsupported message " + m + + ". StreamingVisitor can only accept query result messages."); } ack(token); } @@ -314,38 +309,30 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { handleSummary(summary); } - private void handleSearchResult(SearchResult sr) { - final int hitCountTotal = sr.getTotalHitCount(); - final int hitCount = sr.getHitCount(); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); - } + private void handleSearchResult(SearchResult result) { + log.log(Level.FINE, () -> "Got SearchResult with " + result.getTotalHitCount() + + " in total and " + result.getHitCount() + + " hits in real for query with selection " + params.getDocumentSelection()); + + List<SearchResult.Hit> newHits = new ArrayList<>(result.getHitCount()); + for (int i = 0; i < result.getHitCount(); i++) + newHits.add(result.getHit(i)); - List<SearchResult.Hit> newHits = new ArrayList<>(hitCount); - for (int i = 0; i < hitCount; i++) { - SearchResult.Hit hit = sr.getHit(i); - newHits.add(hit); - } synchronized (this) { - totalHitCount += hitCountTotal; + totalHitCount += result.getTotalHitCount(); hits = ListMerger.mergeIntoArrayList(hits, newHits, query.getOffset() + query.getHits()); } - Map<Integer, byte []> newGroupingMap = sr.getGroupingList(); + Map<Integer, byte[]> newGroupingMap = result.getGroupingList(); mergeGroupingMaps(newGroupingMap); } private void mergeGroupingMaps(Map<Integer, byte []> newGroupingMap) { - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); - } - for(Integer key : newGroupingMap.keySet()) { - byte [] value = newGroupingMap.get(key); - + log.log(Level.FINEST, () -> "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); + for (Integer key : newGroupingMap.keySet()) { + byte[] value = newGroupingMap.get(key); + log.log(Level.FINEST, () ->"Received group with key " + key + " and size " + value.length); Grouping newGrouping = new Grouping(); - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, "Received group with key " + key + " and size " + value.length); - } BufferSerializer buf = new BufferSerializer( new GrowableByteBuffer(ByteBuffer.wrap(value)) ); newGrouping.deserialize(buf); if (buf.getBuf().hasRemaining()) { @@ -366,9 +353,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); - } + log.log(Level.FINE, () -> "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + + params.getDocumentSelection()); synchronized (summaryMap) { for (int i = 0; i < summaryCount; i++) { DocumentSummary.Summary summary = ds.getSummary(i); diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java index 8e2380f72c2..0ebfb9cf669 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java @@ -18,13 +18,14 @@ import static org.junit.jupiter.api.Assertions.*; */ public class MetricsSearcherTestCase { - private MetricsSearcher metricsSearcher = new MetricsSearcher(); - private MockBackend backend = new MockBackend(); - private Chain<Searcher> chain = new Chain<>(metricsSearcher, backend); - private Execution.Context context = Execution.Context.createContextStub(); - private MetricsSearcher.Stats expStatsLt1 = new MetricsSearcher.Stats(); + private final MetricsSearcher metricsSearcher = new MetricsSearcher(); + private final MockBackend backend = new MockBackend(); + private final Chain<Searcher> chain = new Chain<>(metricsSearcher, backend); + private final Execution.Context context = Execution.Context.createContextStub(); + private final MetricsSearcher.Stats expStatsLt1 = new MetricsSearcher.Stats(); + private final MetricsSearcher.Stats expStatsLt2 = new MetricsSearcher.Stats(); + private static final String LOADTYPE1 = "lt1"; - private MetricsSearcher.Stats expStatsLt2 = new MetricsSearcher.Stats(); private static final String LOADTYPE2 = "lt2"; private void verifySearch(String metricParam, String message, String detailedMessage) { @@ -97,8 +98,9 @@ public class MetricsSearcherTestCase { } private static class MockBackend extends Searcher { + private int sequenceNumber = 0; - private VisitorStatistics visitorStats = new VisitorStatistics(); + private final VisitorStatistics visitorStats = new VisitorStatistics(); private boolean implicitlyCreateContext = true; private MockBackend() { @@ -132,9 +134,9 @@ public class MetricsSearcherTestCase { private void assignContextProperties(Query query, String loadType) { if (loadType != null && loadType.equals(LOADTYPE1)) { - query.getContext(true).setProperty(VdsStreamingSearcher.STREAMING_STATISTICS, visitorStats); + query.getContext(true).setProperty(StreamingSearcher.STREAMING_STATISTICS, visitorStats); } else { - query.getContext(true).setProperty(VdsStreamingSearcher.STREAMING_STATISTICS, null); + query.getContext(true).setProperty(StreamingSearcher.STREAMING_STATISTICS, null); } } } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java index 19c03faae66..7b64976cba6 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java @@ -44,7 +44,7 @@ import static org.mockito.Mockito.when; /** * @author Ulf Carlin */ -public class VdsStreamingSearcherTestCase { +public class StreamingSearcherTestCase { public static final String USERDOC_ID_PREFIX = "id:namespace:mytype:n=1:userspecific"; public static final String GROUPDOC_ID_PREFIX = "id:namespace:mytype:g=group1:userspecific"; @@ -98,7 +98,7 @@ public class VdsStreamingSearcherTestCase { groupings.add(new Grouping()); } else if (queryString.compareTo("match_features") == 0) { addResults(USERDOC_ID_PREFIX, 1, false); - var matchFeatures = new MatchFeatureData(Arrays.asList("my_feature")).addHit(); + var matchFeatures = new MatchFeatureData(List.of("my_feature")).addHit(); matchFeatures.set(0, 7.0); hits.get(0).setMatchFeatures(matchFeatures); } @@ -161,7 +161,7 @@ public class VdsStreamingSearcherTestCase { } } - private static Result executeQuery(VdsStreamingSearcher searcher, Query query) { + private static Result executeQuery(StreamingSearcher searcher, Query query) { Execution execution = new Execution(Execution.Context.createContextStub()); return searcher.doSearch2(query, execution); } @@ -184,7 +184,7 @@ public class VdsStreamingSearcherTestCase { return queries; } - private static void checkError(VdsStreamingSearcher searcher, String queryString, String message, String detailedMessage) { + private static void checkError(StreamingSearcher searcher, String queryString, String message, String detailedMessage) { for (Query query : generateTestQueries(queryString)) { Result result = executeQuery(searcher, query); assertNotNull(result.hits().getError()); @@ -197,7 +197,7 @@ public class VdsStreamingSearcherTestCase { } } - private static void checkSearch(VdsStreamingSearcher searcher, String queryString, int hitCount, String idPrefix) { + private static void checkSearch(StreamingSearcher searcher, String queryString, int hitCount, String idPrefix) { for (Query query : generateTestQueries(queryString)) { Result result = executeQuery(searcher, query); assertNull(result.hits().getError()); @@ -215,11 +215,11 @@ public class VdsStreamingSearcherTestCase { } } - private static void checkGrouping(VdsStreamingSearcher searcher, String queryString, int hitCount) { + private static void checkGrouping(StreamingSearcher searcher, String queryString, int hitCount) { checkSearch(searcher, queryString, hitCount, null); } - private static void checkMatchFeatures(VdsStreamingSearcher searcher) { + private static void checkMatchFeatures(StreamingSearcher searcher) { String queryString = "/?streaming.selection=true&query=match_features"; Result result = executeQuery(searcher, new Query(queryString)); assertNull(result.hits().getError()); @@ -232,7 +232,7 @@ public class VdsStreamingSearcherTestCase { @Test void testBasics() { MockVisitorFactory factory = new MockVisitorFactory(); - VdsStreamingSearcher searcher = new VdsStreamingSearcher(factory); + StreamingSearcher searcher = new StreamingSearcher(factory); var schema = new Schema.Builder("test"); schema.add(new com.yahoo.search.schema.DocumentSummary.Builder("default").build()); @@ -281,25 +281,25 @@ public class VdsStreamingSearcherTestCase { String groupId2 = "id:namespace:mytype:g=group2:userspecific"; String badId = "unknowscheme:namespace:something"; - assertTrue(VdsStreamingSearcher.verifyDocId(userId1, generalQuery, true)); - - assertTrue(VdsStreamingSearcher.verifyDocId(userId1, generalQuery, false)); - assertTrue(VdsStreamingSearcher.verifyDocId(userId2, generalQuery, false)); - assertTrue(VdsStreamingSearcher.verifyDocId(groupId1, generalQuery, false)); - assertTrue(VdsStreamingSearcher.verifyDocId(groupId2, generalQuery, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(badId, generalQuery, false)); - - assertTrue(VdsStreamingSearcher.verifyDocId(userId1, user1Query, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(userId2, user1Query, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(groupId1, user1Query, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(groupId2, user1Query, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(badId, user1Query, false)); - - assertFalse(VdsStreamingSearcher.verifyDocId(userId1, group1Query, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(userId2, group1Query, false)); - assertTrue(VdsStreamingSearcher.verifyDocId(groupId1, group1Query, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(groupId2, group1Query, false)); - assertFalse(VdsStreamingSearcher.verifyDocId(badId, group1Query, false)); + assertTrue(StreamingSearcher.verifyDocId(userId1, generalQuery, true)); + + assertTrue(StreamingSearcher.verifyDocId(userId1, generalQuery, false)); + assertTrue(StreamingSearcher.verifyDocId(userId2, generalQuery, false)); + assertTrue(StreamingSearcher.verifyDocId(groupId1, generalQuery, false)); + assertTrue(StreamingSearcher.verifyDocId(groupId2, generalQuery, false)); + assertFalse(StreamingSearcher.verifyDocId(badId, generalQuery, false)); + + assertTrue(StreamingSearcher.verifyDocId(userId1, user1Query, false)); + assertFalse(StreamingSearcher.verifyDocId(userId2, user1Query, false)); + assertFalse(StreamingSearcher.verifyDocId(groupId1, user1Query, false)); + assertFalse(StreamingSearcher.verifyDocId(groupId2, user1Query, false)); + assertFalse(StreamingSearcher.verifyDocId(badId, user1Query, false)); + + assertFalse(StreamingSearcher.verifyDocId(userId1, group1Query, false)); + assertFalse(StreamingSearcher.verifyDocId(userId2, group1Query, false)); + assertTrue(StreamingSearcher.verifyDocId(groupId1, group1Query, false)); + assertFalse(StreamingSearcher.verifyDocId(groupId2, group1Query, false)); + assertFalse(StreamingSearcher.verifyDocId(badId, group1Query, false)); } private static class TraceFixture { @@ -309,13 +309,13 @@ public class VdsStreamingSearcherTestCase { TracingOptions options; MockVisitorFactory factory; - VdsStreamingSearcher searcher; + StreamingSearcher searcher; private TraceFixture(Long firstTimestamp, Long... additionalTimestamps) { clock = MockUtils.mockedClockReturning(firstTimestamp, additionalTimestamps); options = new TracingOptions(sampler, exporter, clock, 8, 2.0); factory = new MockVisitorFactory(); - searcher = new VdsStreamingSearcher(factory, options); + searcher = new StreamingSearcher(factory, options); } private TraceFixture() { diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingVisitorTest.java index 28a34ff2f6d..556d1174ed1 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingVisitorTest.java @@ -20,15 +20,16 @@ import com.yahoo.vespa.objects.BufferSerializer; import org.junit.jupiter.api.Test; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.*; /** - * @author <a href="mailto:ulf@yahoo-inc.com">Ulf Carlin</a> + * @author Ulf Carlin< */ -public class VdsVisitorTestCase { +public class StreamingVisitorTest { private SearchResult createSR(String docId, double rank) { BufferSerializer serializer = new BufferSerializer(); @@ -149,7 +150,7 @@ public class VdsVisitorTestCase { queryString.append("&location=").append(qa.location); } if (qa.sortSpec != null) { - queryString.append("&sorting=").append(URLEncoder.encode(qa.sortSpec, "UTF-8")); + queryString.append("&sorting=").append(URLEncoder.encode(qa.sortSpec, StandardCharsets.UTF_8)); } if (qa.rankProperties != null) { queryString.append("&").append(qa.rankProperties); @@ -162,7 +163,7 @@ public class VdsVisitorTestCase { queryString.append("&streaming.groupname=").append(qa.groupName); } if (qa.selection != null) { - queryString.append("&streaming.selection=").append(URLEncoder.encode(qa.selection, "UTF-8")); + queryString.append("&streaming.selection=").append(URLEncoder.encode(qa.selection, StandardCharsets.UTF_8)); } if (qa.from != 0) { queryString.append("&streaming.fromtimestamp=").append(qa.from); @@ -209,7 +210,7 @@ public class VdsVisitorTestCase { if (qa.maxBucketsPerVisitor != 0) { assertEquals(qa.maxBucketsPerVisitor, params.getMaxBucketsPerVisitor()); } else { - assertEquals(VdsVisitor.MAX_BUCKETS_PER_VISITOR, params.getMaxBucketsPerVisitor()); + assertEquals(StreamingVisitor.MAX_BUCKETS_PER_VISITOR, params.getMaxBucketsPerVisitor()); } // Verify parameters based only on query @@ -267,17 +268,17 @@ public class VdsVisitorTestCase { @Test void testGetQueryFlags() { - assertEquals(0x00028000, VdsVisitor.getQueryFlags(new Query("/?query=test"))); - assertEquals(0x00028080, VdsVisitor.getQueryFlags(new Query("/?query=test&hitcountestimate=true"))); - assertEquals(0x00068000, VdsVisitor.getQueryFlags(new Query("/?query=test&rankfeatures=true"))); - assertEquals(0x00068080, VdsVisitor.getQueryFlags(new Query("/?query=test&hitcountestimate=true&rankfeatures=true"))); + assertEquals(0x00028000, StreamingVisitor.getQueryFlags(new Query("/?query=test"))); + assertEquals(0x00028080, StreamingVisitor.getQueryFlags(new Query("/?query=test&hitcountestimate=true"))); + assertEquals(0x00068000, StreamingVisitor.getQueryFlags(new Query("/?query=test&rankfeatures=true"))); + assertEquals(0x00068080, StreamingVisitor.getQueryFlags(new Query("/?query=test&hitcountestimate=true&rankfeatures=true"))); Query query = new Query("/?query=test"); - assertEquals(0x00028000, VdsVisitor.getQueryFlags(query)); + assertEquals(0x00028000, StreamingVisitor.getQueryFlags(query)); query.setNoCache(true); - assertEquals(0x00038000, VdsVisitor.getQueryFlags(query)); + assertEquals(0x00038000, StreamingVisitor.getQueryFlags(query)); query.getRanking().setFreshness("now"); - assertEquals(0x0003a000, VdsVisitor.getQueryFlags(query)); + assertEquals(0x0003a000, StreamingVisitor.getQueryFlags(query)); } @Test @@ -321,7 +322,7 @@ public class VdsVisitorTestCase { } private void verifyVisitorOk(MockVisitorSessionFactory factory, QueryArguments qa, Route route, String searchCluster) throws Exception { - VdsVisitor visitor = new VdsVisitor(buildQuery(qa), searchCluster, route, "mytype", factory, 0); + StreamingVisitor visitor = new StreamingVisitor(buildQuery(qa), searchCluster, route, "mytype", factory, 0); visitor.doSearch(); verifyVisitorParameters(factory.getParams(), qa, searchCluster, "mytype", route); supplyResults(visitor); @@ -329,10 +330,10 @@ public class VdsVisitorTestCase { } private void verifyVisitorFails(MockVisitorSessionFactory factory, QueryArguments qa, Route route, String searchCluster) throws Exception { - VdsVisitor visitor = new VdsVisitor(buildQuery(qa), searchCluster, route, "mytype", factory, 0); + StreamingVisitor visitor = new StreamingVisitor(buildQuery(qa), searchCluster, route, "mytype", factory, 0); try { visitor.doSearch(); - assertTrue(false, "Visitor did not fail"); + fail("Visitor did not fail"); } catch (TimeoutException te) { assertTrue(factory.timeoutQuery, "Got TimeoutException unexpectedly"); } catch (IllegalArgumentException iae) { @@ -340,20 +341,20 @@ public class VdsVisitorTestCase { } } - private void supplyResults(VdsVisitor visitor) { + private void supplyResults(StreamingVisitor visitor) { AckToken ackToken = null; visitor.onMessage(createQRM("id:ns:type::0", 0.3), ackToken); visitor.onMessage(createQRM("id:ns:type::1", 1.0), ackToken); visitor.onMessage(createQRM("id:ns:type::2", 0.5), ackToken); try { visitor.onMessage(createM(), ackToken); - assertTrue(false, "Unsupported message did not cause exception"); + fail("Unsupported message did not cause exception"); } catch (UnsupportedOperationException uoe) { - assertTrue(uoe.getMessage().contains("VdsVisitor can only accept query result messages")); + assertTrue(uoe.getMessage().contains("StreamingVisitor can only accept query result messages")); } } - private void verifyResults(QueryArguments qa, VdsVisitor visitor) { + private void verifyResults(QueryArguments qa, StreamingVisitor visitor) { assertEquals(6, visitor.getTotalHitCount()); assertEquals(Math.min(3 - qa.offset, qa.hits), visitor.getHits().size()); assertEquals(3, visitor.getSummaryMap().size()); @@ -373,7 +374,7 @@ public class VdsVisitorTestCase { assertEquals("id:ns:type::0", hit.getDocId()); assertEquals(0.3, hit.getRank(), 0.01); } else { - assertTrue(false, "Got too many hits"); + fail("Got too many hits"); } DocumentSummary.Summary summary = visitor.getSummaryMap().get(hit.getDocId()); assertNotNull(summary, "Did not find summary for " + hit.getDocId()); @@ -381,9 +382,10 @@ public class VdsVisitorTestCase { } private static class MockVisitorSession implements VisitorSession { - private VisitorParameters params; - private boolean timeoutQuery = false; - private boolean failQuery = false; + + private final VisitorParameters params; + private final boolean timeoutQuery; + private final boolean failQuery; public MockVisitorSession(VisitorParameters params, boolean timeoutQuery, boolean failQuery) { this.params = params; @@ -444,7 +446,8 @@ public class VdsVisitorTestCase { } } - private static class MockVisitorSessionFactory implements VdsVisitor.VisitorSessionFactory { + private static class MockVisitorSessionFactory implements StreamingVisitor.VisitorSessionFactory { + private VisitorParameters params; private boolean timeoutQuery = false; private boolean failQuery = false; |