diff options
author | Jon Bratseth <jonbratseth@yahoo.com> | 2018-02-07 13:55:26 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-02-07 13:55:26 +0100 |
commit | 4edbbfb6c3e9be923137b7ea9cf039bf7a19c8a4 (patch) | |
tree | 2aa84005557c96f48a40323827750f194598a30c | |
parent | 521be18377a117c5d02c1dd104017457c375f25b (diff) | |
parent | 553575a45c8697b247036c5a4a6b2e9e9fb862ee (diff) |
Merge pull request #4949 from vespa-engine/balder/cleanup-softtimeout
Balder/cleanup softtimeout
27 files changed, 272 insertions, 219 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index d714d0211ab..481e3f5ba97 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -17,6 +17,7 @@ public class Coverage { protected long soonActive; protected int degradedReason; protected int nodes; + private int nodesTried; protected int resultSets; protected int fullResultSets; @@ -34,8 +35,7 @@ public class Coverage { /** * Build an invalid instance to initiate manually. */ - protected Coverage() { - } + protected Coverage() { } protected Coverage(long docs, long active, int nodes, int resultSets) { this(docs, active, nodes, resultSets, FullCoverageDefinition.DOCUMENT_COUNT); @@ -45,7 +45,7 @@ public class Coverage { this(docs, nodes, full, 1); } - public Coverage(long docs, int nodes, boolean full, int resultSets) { + protected Coverage(long docs, int nodes, boolean full, int resultSets) { this(docs, docs, nodes, resultSets, full ? FullCoverageDefinition.EXPLICITLY_FULL : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); } @@ -53,6 +53,7 @@ public class Coverage { private Coverage(long docs, long active, int nodes, int resultSets, FullCoverageDefinition fullReason) { this.docs = docs; this.nodes = nodes; + this.nodesTried = nodes; this.active = active; this.soonActive = active; this.degradedReason = 0; @@ -67,6 +68,7 @@ public class Coverage { } docs += other.getDocs(); nodes += other.getNodes(); + nodesTried += other.nodesTried; active += other.getActive(); soonActive += other.getSoonActive(); degradedReason |= other.degradedReason; @@ -137,13 +139,22 @@ public class Coverage { } /** - * @return the number of search instances which participated in the search. + * @return the number of search instances which participated successfully in the search. */ public int getNodes() { return nodes; } /** + * @return the number of search instances which tried to participate in the search. + */ + public int getNodesTried() { + return nodesTried; + } + + public Coverage setNodesTried(int nodesTried) { this.nodesTried = nodesTried; return this; } + + /** * A Coverage instance contains coverage information for potentially more * than one search. If several queries, e.g. through blending of results * from multiple clusters, produced a result set, this number will show how diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index 8a14ea5c343..0709baa9be2 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -217,6 +217,7 @@ public class QueryPacket extends Packet { * query flag bits, taken from searchlib/common/transport.h **/ static final int QFLAG_EXTENDED_COVERAGE = 0x00000001; + static final int QFLAG_COVERAGE_NODES = 0x00000002; static final int QFLAG_ESTIMATE = 0x00000080; static final int QFLAG_DROP_SORTDATA = 0x00004000; static final int QFLAG_NO_RESULTCACHE = 0x00010000; @@ -250,7 +251,7 @@ public class QueryPacket extends Packet { } static int getQueryFlags(Query query) { - int flags = QFLAG_EXTENDED_COVERAGE; + int flags = QFLAG_EXTENDED_COVERAGE | QFLAG_COVERAGE_NODES; flags |= query.properties().getBoolean(com.yahoo.search.query.Model.ESTIMATE) ? QFLAG_ESTIMATE : 0; flags |= query.getNoCache() ? QFLAG_NO_RESULTCACHE : 0; diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java index 17f3ca2655f..45a06f6b4f3 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java @@ -1,14 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.fs4; -import com.yahoo.search.Query; - import java.nio.ByteBuffer; import java.nio.IntBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.logging.Logger; /** @@ -26,16 +23,14 @@ public class QueryResultPacket extends Packet { /** Whether mld stuff, whatever that is, is included in this result */ private boolean mldFeature=false; - /** A feature of no apparent utility */ - private boolean datasetFeature=false; - /** Whether coverage information is included in this result */ - private boolean coverageFeature = false; - private boolean coverageExtendedFeature = false; - private long coverageDocs = 0; - private long activeDocs = 0; - private long soonActiveDocs = 0; - private int degradedReason = 0; + private boolean coverageNodes = false; + private long coverageDocs = 0; + private long activeDocs = 0; + private long soonActiveDocs = 0; + private int degradedReason = 0; + private short nodesQueried = 0; + private short nodesReplied = 0; /** Whether the result contains grouping results **/ private boolean groupDataFeature = false; @@ -49,8 +44,6 @@ public class QueryResultPacket extends Packet { private int docstamp; - private int dataset=-1; - private byte[] groupData = null; private List<DocumentInfo> documents=new ArrayList<>(10); @@ -59,8 +52,7 @@ public class QueryResultPacket extends Packet { private int offset; - private QueryResultPacket() { - } + private QueryResultPacket() { } public static QueryResultPacket create() { return new QueryResultPacket(); @@ -73,12 +65,7 @@ public class QueryResultPacket extends Packet { /** Returns whether this has the mysterious mld feature */ public boolean getMldFeature() { return mldFeature; } - /** Returns whether this result has the dataset feature */ - public boolean getDatasetFeature() { return datasetFeature; } - - public boolean getCoverageFeature() { - return coverageFeature; - } + public boolean getCoverageFeature() { return true; } public long getCoverageDocs() { return coverageDocs; } @@ -113,24 +100,22 @@ public class QueryResultPacket extends Packet { maxRank = decodeMaxRank(buffer); ints = buffer.asIntBuffer(); docstamp=ints.get(); - if (datasetFeature) dataset=ints.get(); buffer.position(buffer.position() + ints.position()*4); + if (coverageNodes) { + nodesQueried = buffer.getShort(); + nodesReplied = buffer.getShort(); + } if (groupDataFeature) { int len = buffer.getInt(); groupData = new byte[len]; buffer.get(groupData); } - if (coverageFeature) { - coverageDocs = buffer.getLong(); - activeDocs = buffer.getLong(); - } - if (coverageExtendedFeature) { - soonActiveDocs = buffer.getLong(); - degradedReason = buffer.getInt(); - } else { - soonActiveDocs = activeDocs; - degradedReason = 0; - } + + coverageDocs = buffer.getLong(); + activeDocs = buffer.getLong(); + soonActiveDocs = buffer.getLong(); + degradedReason = buffer.getInt(); + decodeDocuments(buffer,documentCount); if (propsFeature) { int numMaps = buffer.getInt(); @@ -149,12 +134,13 @@ public class QueryResultPacket extends Packet { /** * feature bits */ - public static final int QRF_MLD = 0x00000001; - public static final int QRF_SORTDATA = 0x00000010; - public static final int QRF_EXTENDED_COVERAGE = 0x00000020; - public static final int QRF_COVERAGE = 0x00000040; - public static final int QRF_GROUPDATA = 0x00000200; - public static final int QRF_PROPERTIES = 0x00000400; + public static final int QRF_MLD = 0x00000001; + public static final int QRF_COVERAGE_NODES = 0x00000002; + public static final int QRF_SORTDATA = 0x00000010; + public static final int QRF_UNUSED_1 = 0x00000020; + public static final int QRF_UNUSED_2 = 0x00000040; + public static final int QRF_GROUPDATA = 0x00000200; + public static final int QRF_PROPERTIES = 0x00000400; /** * Sets the features of this package. @@ -166,16 +152,14 @@ public class QueryResultPacket extends Packet { case 217: int features=buffer.get(); mldFeature = (QRF_MLD & features) != 0; - datasetFeature = (0x002 & features) != 0; // Data given by sortFeature not currently used by QRS: // sortFeature = (QRF_SORTDATA & features) != 0; - coverageExtendedFeature = (QRF_EXTENDED_COVERAGE & features) != 0; - coverageFeature = (QRF_COVERAGE & features) != 0; + coverageNodes = (QRF_COVERAGE_NODES & features) != 0; groupDataFeature = (QRF_GROUPDATA & features) != 0; propsFeature = (QRF_PROPERTIES & features) != 0; break; default: - throw new RuntimeException("Programming error"); + throw new RuntimeException("Programming error, packet " + getCode() + "Not expected."); } } @@ -219,6 +203,7 @@ public class QueryResultPacket extends Packet { // TODO: Handle new maxRank intelligently public int getMaxRank() { return maxRank.intValue(); } - public int getDataset() { return dataset; } + public short getNodesQueried() { return nodesQueried; } + public short getNodesReplied() { return nodesReplied; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index ea354e1291f..3320c167702 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -465,9 +465,10 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { } if (resultPacket.getCoverageFeature()) { - result.setCoverage(new Coverage(resultPacket.getCoverageDocs(), resultPacket.getActiveDocs()) + result.setCoverage(new Coverage(resultPacket.getCoverageDocs(), resultPacket.getActiveDocs(), resultPacket.getNodesReplied()) .setSoonActive(resultPacket.getSoonActiveDocs()) - .setDegradedReason(resultPacket.getDegradedReason())); + .setDegradedReason(resultPacket.getDegradedReason()) + .setNodesTried(resultPacket.getNodesQueried())); } } diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index a015344780f..81aa4a08f18 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -12,13 +12,19 @@ import com.google.common.annotations.Beta; public class Coverage extends com.yahoo.container.handler.Coverage { public Coverage(long docs, long active) { - super(docs, active, 0, 1); + this(docs, active, 0); } + public Coverage(long docs, long active, int nodes) { + super(docs, active, nodes, 1); + } + + @Deprecated public Coverage(long docs, int nodes, boolean full) { this(docs, nodes, full, 1); } + @Deprecated public Coverage(long docs, int nodes, boolean full, int resultSets) { super(docs, nodes, full, resultSets); } @@ -38,4 +44,6 @@ public class Coverage extends com.yahoo.container.handler.Coverage { */ public Coverage setDegradedReason(int degradedReason) { this.degradedReason = degradedReason; return this; } + public Coverage setNodesTried(int nodesTried) { super.setNodesTried(nodesTried); return this; } + } diff --git a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java index b771c64d1a9..679bc96c06b 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java @@ -36,13 +36,13 @@ public class GetDocSumsPacketTestCase { FastHit hit = new FastHit(); hit.setIgnoreRowBits(true); assertPacket(true, hit, new byte[] { 0, 0, 0, 57, 0, 0, 0, -37, 0, 0, 40, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, - IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7, + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x03, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111, 0, 0, 0, 3 }); hit = new FastHit(); hit.setIgnoreRowBits(false); assertPacket(true, hit, new byte[] {0, 0, 0, 57, 0, 0, 0, -37, 0, 0, 40, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, - IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7, + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x03, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111, 0, 0, 0, 2}); } @@ -51,14 +51,14 @@ public class GetDocSumsPacketTestCase { FastHit hit = new FastHit(); hit.setIgnoreRowBits(true); assertPacket(false, hit, new byte[] { 0, 0, 0, 43, 0, 0, 0, -37, 0, 0, 40, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, - IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7, + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x03, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 3 }); hit = new FastHit(); hit.setIgnoreRowBits(false); assertPacket(false, hit, new byte[] { 0, 0, 0, 43, 0, 0, 0, -37, 0, 0, 40, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, - IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 2 + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x03, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 2 }); } @@ -75,7 +75,7 @@ public class GetDocSumsPacketTestCase { // query timeout IGNORE, IGNORE, IGNORE, IGNORE, // "default" - rank profile - 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 0x01, + 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 0x03, // "default" - summaryclass 0, 0, 0, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', // 2 property entries diff --git a/container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java index cdd2dc7e921..a5712f625e1 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java @@ -24,18 +24,23 @@ public class QueryResultTestCase extends junit.framework.TestCase { private static GlobalId gid2 = new GlobalId(new byte[] {2,2,2,2,2,2,2,2,2,2,2,2}); public void testDecodeQueryResultX() { - byte[] packetData=new byte[] {0,0,0,100, - 0,0,0,217-256, - 0,0,0,1, - 0,0,0,15, - 0,0,0,0, - 0,0,0,2, - 0,0,0,0,0,0,0,5, - 0x40,0x39,0,0,0,0,0,0, - 0,0,0,111, - 0,0,0,97, - 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, - 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37}; + byte[] packetData=new byte[] { + 0,0,0,100, + 0,0,0,217-256, + 0,0,0,1, + 0,0,0,1, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 0,0,0,0,0,0,0,89, + 0,0,0,0,0,0,0,90, + 0,0,0,0,0,0,0,91, + 0,0,0,1, + 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, + 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37 + }; ByteBuffer buffer=ByteBuffer.allocate(200); buffer.put(packetData); buffer.flip(); @@ -44,12 +49,14 @@ public class QueryResultTestCase extends junit.framework.TestCase { QueryResultPacket result=(QueryResultPacket)packet; assertTrue(result.getMldFeature()); - assertTrue(result.getDatasetFeature()); assertEquals(5,result.getTotalDocumentCount()); assertEquals(25,result.getMaxRank()); assertEquals(111,result.getDocstamp()); - assertEquals(97,result.getDataset()); + assertEquals(89, result.getCoverageDocs()); + assertEquals(90, result.getActiveDocs()); + assertEquals(91, result.getSoonActiveDocs()); + assertEquals(1, result.getDegradedReason()); assertEquals(2,result.getDocuments().size()); DocumentInfo document1= result.getDocuments().get(0); @@ -65,18 +72,24 @@ public class QueryResultTestCase extends junit.framework.TestCase { } public void testDecodeQueryResultMoreHits() { - byte[] packetData=new byte[] {0,0,0,100, - 0,0,0,217-256, - 0,0,0,1, - 0,0,0,15, - 0,0,0,0, - 0,0,0,2, - 0,0,0,0,0,0,0,5, - 0x40,0x39,0,0,0,0,0,0, - 0,0,0,111, - 0,0,0,97, - 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, - 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37}; + byte[] packetData=new byte[] { + 0,0,0,100, + 0,0,0,217-256, + 0,0,0,1, + 0,0,0,3, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 0,6,0,5, + 0,0,0,0,0,0,0,89, + 0,0,0,0,0,0,0,90, + 0,0,0,0,0,0,0,91, + 0,0,0,1, + 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, + 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37 + }; ByteBuffer buffer=ByteBuffer.allocate(200); buffer.put(packetData); buffer.flip(); @@ -89,5 +102,7 @@ public class QueryResultTestCase extends junit.framework.TestCase { assertEquals(gid1,document1.getGlobalId()); DocumentInfo document2= result.getDocuments().get(1); assertEquals(gid2,document2.getGlobalId()); + assertEquals(6, result.getNodesQueried()); + assertEquals(5, result.getNodesReplied()); } } diff --git a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java index c7359ce2f94..603e38e04f1 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java @@ -36,7 +36,7 @@ public class QueryTestCase extends junit.framework.TestCase { 2, 8, 0,0,0,1, // querytimeout - 0,0,0x40,0x01, // qflags + 0,0,0x40,0x03, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0,0,0,1,0,0,0,8,4, @@ -58,7 +58,7 @@ public class QueryTestCase extends junit.framework.TestCase { 2, 8, 0,0,0,1, // querytimeout - 0,0,0x40,0x01, // QFlags + 0,0,0x40,0x03, // QFlags 3, 't','w','o', // Ranking 0,0,0,1,0,0,0,8,4, @@ -88,7 +88,7 @@ public class QueryTestCase extends junit.framework.TestCase { byte[] encoded = packetToBytes(packet); // System.out.println(Arrays.toString(encoded)); byte[] correctBuffer=new byte[] { - 0, 0, 1, 23, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, -122, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x01, 3, 't', 'w', 'o', 0, 0, 0, 3, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '2', 0, 0, 0, 2, 'v', '2', 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '1', 0, 0, 0, 2, 'v', '1', 0, 0, 0, 3, 'f', 'o', 'o', 0, 0, 0, 4, '3', '0', 46, '3', 0, 0, 0, 3, 'b', 'a', 'r', 0, 0, 0, 1, '0', 0, 0, 0, 9, 'v', 'e', 's', 'p', 'a', 46, 'n', 'o', 'w', 0, 0, 0, 6, '1', '2', '3', '4', '5', '6', 0, 0, 0, 14, 'h', 'i', 'g', 'h', 'l', 'i', 'g', 'h', 't', 't', 'e', 'r', 'm', 's', 0, 0, 0, 3, 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 1, '2', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '1', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '2', 0, 0, 0, 5, 'm', 'o', 'd', 'e', 'l', 0, 0, 0, 1, 0, 0, 0, 10, 's', 'e', 'a', 'r', 'c', 'h', 'p', 'a', 't', 'h', 0, 0, 0, 3, '7', 47, '3', 0, 0, 0, 15, 43, 'f', 'i', 'e', 'l', 'd', '1', 32, 45, 'f', 'i', 'e', 'l', 'd', '2', 0, 0, 0, 1, 0, 0, 0, 9, 68, 1, 0, 5, 'c', 'h', 'a', 'i', 'n' + 0, 0, 1, 23, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, -122, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x03, 3, 't', 'w', 'o', 0, 0, 0, 3, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '2', 0, 0, 0, 2, 'v', '2', 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '1', 0, 0, 0, 2, 'v', '1', 0, 0, 0, 3, 'f', 'o', 'o', 0, 0, 0, 4, '3', '0', 46, '3', 0, 0, 0, 3, 'b', 'a', 'r', 0, 0, 0, 1, '0', 0, 0, 0, 9, 'v', 'e', 's', 'p', 'a', 46, 'n', 'o', 'w', 0, 0, 0, 6, '1', '2', '3', '4', '5', '6', 0, 0, 0, 14, 'h', 'i', 'g', 'h', 'l', 'i', 'g', 'h', 't', 't', 'e', 'r', 'm', 's', 0, 0, 0, 3, 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 1, '2', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '1', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '2', 0, 0, 0, 5, 'm', 'o', 'd', 'e', 'l', 0, 0, 0, 1, 0, 0, 0, 10, 's', 'e', 'a', 'r', 'c', 'h', 'p', 'a', 't', 'h', 0, 0, 0, 3, '7', 47, '3', 0, 0, 0, 15, 43, 'f', 'i', 'e', 'l', 'd', '1', 32, 45, 'f', 'i', 'e', 'l', 'd', '2', 0, 0, 0, 1, 0, 0, 0, 9, 68, 1, 0, 5, 'c', 'h', 'a', 'i', 'n' }; assertEqualArrays(correctBuffer,encoded); } @@ -114,7 +114,7 @@ public class QueryTestCase extends junit.framework.TestCase { byte[] encoded = packetToBytes(packet); // System.out.println(Arrays.toString(encoded)); byte[] correctBuffer=new byte[] { - 0, 0, 1, 23, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, -122, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x01, 3, 't', 'w', 'o', 0, 0, 0, 3, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '2', 0, 0, 0, 2, 'v', '2', 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '1', 0, 0, 0, 2, 'v', '1', 0, 0, 0, 3, 'f', 'o', 'o', 0, 0, 0, 4, '3', '0', 46, '3', 0, 0, 0, 3, 'b', 'a', 'r', 0, 0, 0, 1, '0', 0, 0, 0, 9, 'v', 'e', 's', 'p', 'a', 46, 'n', 'o', 'w', 0, 0, 0, 6, '1', '2', '3', '4', '5', '6', 0, 0, 0, 14, 'h', 'i', 'g', 'h', 'l', 'i', 'g', 'h', 't', 't', 'e', 'r', 'm', 's', 0, 0, 0, 3, 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 1, '2', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '1', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '2', 0, 0, 0, 5, 'm', 'o', 'd', 'e', 'l', 0, 0, 0, 1, 0, 0, 0, 10, 's', 'e', 'a', 'r', 'c', 'h', 'p', 'a', 't', 'h', 0, 0, 0, 3, '7', 47, '3', 0, 0, 0, 15, 43, 'f', 'i', 'e', 'l', 'd', '1', 32, 45, 'f', 'i', 'e', 'l', 'd', '2', 0, 0, 0, 1, 0, 0, 0, 9, 68, 1, 0, 5, 'c', 'h', 'a', 'i', 'n' + 0, 0, 1, 23, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, -122, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x03, 3, 't', 'w', 'o', 0, 0, 0, 3, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '2', 0, 0, 0, 2, 'v', '2', 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '1', 0, 0, 0, 2, 'v', '1', 0, 0, 0, 3, 'f', 'o', 'o', 0, 0, 0, 4, '3', '0', 46, '3', 0, 0, 0, 3, 'b', 'a', 'r', 0, 0, 0, 1, '0', 0, 0, 0, 9, 'v', 'e', 's', 'p', 'a', 46, 'n', 'o', 'w', 0, 0, 0, 6, '1', '2', '3', '4', '5', '6', 0, 0, 0, 14, 'h', 'i', 'g', 'h', 'l', 'i', 'g', 'h', 't', 't', 'e', 'r', 'm', 's', 0, 0, 0, 3, 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 1, '2', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '1', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '2', 0, 0, 0, 5, 'm', 'o', 'd', 'e', 'l', 0, 0, 0, 1, 0, 0, 0, 10, 's', 'e', 'a', 'r', 'c', 'h', 'p', 'a', 't', 'h', 0, 0, 0, 3, '7', 47, '3', 0, 0, 0, 15, 43, 'f', 'i', 'e', 'l', 'd', '1', 32, 45, 'f', 'i', 'e', 'l', 'd', '2', 0, 0, 0, 1, 0, 0, 0, 9, 68, 1, 0, 5, 'c', 'h', 'a', 'i', 'n' }; assertEqualArrays(correctBuffer,encoded); } @@ -143,7 +143,7 @@ public class QueryTestCase extends junit.framework.TestCase { QueryPacket packet=QueryPacket.create(query); byte[] encoded = packetToBytes(packet); byte[] correctBuffer=new byte[] { - 0, 0, 1, 16, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x01, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 1, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 18, 'v', 'e', 's', 'p', 'a', 46, 'l', 'a', 'b', 'e', 'l', 46, 'b', 'a', 'r', 46, 'i', 'd', 0, 0, 0, 1, '1', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 1, '3', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 4, '0', 46, '1', '5', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '3', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '3', '7', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '8', '1', 0, 0, 0, 5, 0, 0, 0, '4', 1, 4, 79, 1, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 79, 2, 0, 6, 'f', 'i', 'e', 'l', 'd', '2', 68, 3, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', '1', 68, 4, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', 49 + 0, 0, 1, 16, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x03, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 1, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 18, 'v', 'e', 's', 'p', 'a', 46, 'l', 'a', 'b', 'e', 'l', 46, 'b', 'a', 'r', 46, 'i', 'd', 0, 0, 0, 1, '1', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 1, '3', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 4, '0', 46, '1', '5', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '3', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '3', '7', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '8', '1', 0, 0, 0, 5, 0, 0, 0, '4', 1, 4, 79, 1, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 79, 2, 0, 6, 'f', 'i', 'e', 'l', 'd', '2', 68, 3, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', '1', 68, 4, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', 49 }; assertEqualArrays(correctBuffer,encoded); } @@ -163,7 +163,7 @@ public class QueryTestCase extends junit.framework.TestCase { 2, // offset 8, // maxhits 0,0,0,1, // querytimeout - 0,0,0x40,0x01, // qflags + 0,0,0x40,0x03, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0,0,0,5, // sortspec length @@ -234,7 +234,7 @@ public class QueryTestCase extends junit.framework.TestCase { 2, 8, 0,0,0,1, // querytimeout - 0,0,0x40,0x01, // qflags + 0,0,0x40,0x03, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0,0,0,1,0,0,0,8,4, @@ -253,7 +253,7 @@ public class QueryTestCase extends junit.framework.TestCase { 2, 8, 0,0,0,1, // querytimeout - 0,0,0x40,0x01, // qflags + 0,0,0x40,0x03, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0,0,0,1,0,0,0,8,4, diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 75d965c6438..20150fc2671 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -159,7 +159,7 @@ public class FastSearcherTestCase { byte[] correct = new byte[] { 0, 0, 0, 100, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10, QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, // time left - 0, 0, 0x40, 0x01, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 5, 109, 97, 116, 99, 104, 0, 0, 0, 1, 0, 0, 0, 24, 100, 111, 99, 117, 109, 101, 110, 116, 100, 98, 46, 115, 101, 97, 114, 99, 104, 100, 111, 99, 116, 121, 112, 101, 0, 0, 0, 6, 116, 101, 115, 116, 68, 98, 0, 0, 0, 1, 0, 0, 0, 7, 68, 1, 0, 3, 102, 111, 111 + 0, 0, 0x40, 0x03, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 5, 109, 97, 116, 99, 104, 0, 0, 0, 1, 0, 0, 0, 24, 100, 111, 99, 117, 109, 101, 110, 116, 100, 98, 46, 115, 101, 97, 114, 99, 104, 100, 111, 99, 116, 121, 112, 101, 0, 0, 0, 6, 116, 101, 115, 116, 68, 98, 0, 0, 0, 1, 0, 0, 0, 7, 68, 1, 0, 3, 102, 111, 111 }; QueryTestCase.assertEqualArrays(correct, encoded); } @@ -303,7 +303,7 @@ public class FastSearcherTestCase { // query timeout IGNORE, IGNORE, IGNORE, IGNORE, // "default" - rank profile - 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 0x01, + 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 0x03, // 3 property entries (rank, match, caches) 0, 0, 0, 3, // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0 diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp index 46cba2d59c4..b94026e689c 100644 --- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp @@ -1,13 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("matching_stats_test"); + #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchcore/proton/matching/matching_stats.h> +#include <vespa/log/log.h> +LOG_SETUP("matching_stats_test"); + using namespace proton::matching; TEST("requireThatDocCountsAddUp") { MatchingStats stats; + EXPECT_EQUAL(0u, stats.docsCovered()); EXPECT_EQUAL(0u, stats.docsMatched()); EXPECT_EQUAL(0u, stats.docsRanked()); EXPECT_EQUAL(0u, stats.docsReRanked()); @@ -15,6 +18,7 @@ TEST("requireThatDocCountsAddUp") { EXPECT_EQUAL(0u, stats.limited_queries()); { MatchingStats rhs; + EXPECT_EQUAL(&rhs.docsCovered(10000), &rhs); EXPECT_EQUAL(&rhs.docsMatched(1000), &rhs); EXPECT_EQUAL(&rhs.docsRanked(100), &rhs); EXPECT_EQUAL(&rhs.docsReRanked(10), &rhs); @@ -22,13 +26,15 @@ TEST("requireThatDocCountsAddUp") { EXPECT_EQUAL(&rhs.limited_queries(1), &rhs); EXPECT_EQUAL(&stats.add(rhs), &stats); } + EXPECT_EQUAL(10000u, stats.docsCovered()); EXPECT_EQUAL(1000u, stats.docsMatched()); EXPECT_EQUAL(100u, stats.docsRanked()); EXPECT_EQUAL(10u, stats.docsReRanked()); EXPECT_EQUAL(2u, stats.queries()); EXPECT_EQUAL(1u, stats.limited_queries()); - EXPECT_EQUAL(&stats.add(MatchingStats().docsMatched(1000).docsRanked(100) + EXPECT_EQUAL(&stats.add(MatchingStats().docsCovered(10000).docsMatched(1000).docsRanked(100) .docsReRanked(10).queries(2).limited_queries(1)), &stats); + EXPECT_EQUAL(20000u, stats.docsCovered()); EXPECT_EQUAL(2000u, stats.docsMatched()); EXPECT_EQUAL(200u, stats.docsRanked()); EXPECT_EQUAL(20u, stats.docsReRanked()); @@ -84,12 +90,14 @@ TEST("requireThatAverageTimesAreRecorded") { TEST("requireThatPartitionsAreAddedCorrectly") { MatchingStats all1; + EXPECT_EQUAL(0u, all1.docsCovered()); EXPECT_EQUAL(0u, all1.docsMatched()); EXPECT_EQUAL(0u, all1.getNumPartitions()); MatchingStats::Partition subPart; - subPart.docsMatched(3).docsRanked(2).docsReRanked(1) + subPart.docsCovered(7).docsMatched(3).docsRanked(2).docsReRanked(1) .active_time(1.0).wait_time(0.5); + EXPECT_EQUAL(7u, subPart.docsCovered()); EXPECT_EQUAL(3u, subPart.docsMatched()); EXPECT_EQUAL(2u, subPart.docsRanked()); EXPECT_EQUAL(1u, subPart.docsReRanked()); @@ -99,10 +107,12 @@ TEST("requireThatPartitionsAreAddedCorrectly") { EXPECT_EQUAL(1u, subPart.wait_time_count()); all1.merge_partition(subPart, 0); + EXPECT_EQUAL(7u, all1.docsCovered()); EXPECT_EQUAL(3u, all1.docsMatched()); EXPECT_EQUAL(2u, all1.docsRanked()); EXPECT_EQUAL(1u, all1.docsReRanked()); EXPECT_EQUAL(1u, all1.getNumPartitions()); + EXPECT_EQUAL(7u, all1.getPartition(0).docsCovered()); EXPECT_EQUAL(3u, all1.getPartition(0).docsMatched()); EXPECT_EQUAL(2u, all1.getPartition(0).docsRanked()); EXPECT_EQUAL(1u, all1.getPartition(0).docsReRanked()); @@ -112,6 +122,7 @@ TEST("requireThatPartitionsAreAddedCorrectly") { EXPECT_EQUAL(1u, all1.getPartition(0).wait_time_count()); all1.merge_partition(subPart, 1); + EXPECT_EQUAL(14u, all1.docsCovered()); EXPECT_EQUAL(6u, all1.docsMatched()); EXPECT_EQUAL(4u, all1.docsRanked()); EXPECT_EQUAL(2u, all1.docsReRanked()); @@ -125,6 +136,7 @@ TEST("requireThatPartitionsAreAddedCorrectly") { EXPECT_EQUAL(1u, all1.getPartition(1).wait_time_count()); all1.add(all1); + EXPECT_EQUAL(28u, all1.docsCovered()); EXPECT_EQUAL(12u, all1.docsMatched()); EXPECT_EQUAL(8u, all1.docsRanked()); EXPECT_EQUAL(4u, all1.docsReRanked()); diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.h b/searchcore/src/vespa/searchcore/fdispatch/common/search.h index 8ca63838c5f..a1a8eb9f0ba 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/search.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/search.h @@ -100,6 +100,8 @@ public: uint64_t _activeDocs; uint64_t _soonActiveDocs; uint32_t _degradeReason; + uint16_t _nodesQueried; + uint16_t _nodesReplied; FastS_SearchInfo() : _searchOffset(0), @@ -107,9 +109,10 @@ public: _coverageDocs(0), _activeDocs(0), _soonActiveDocs(0), - _degradeReason(0) - { - } + _degradeReason(0), + _nodesQueried(0), + _nodesReplied(0) + { } }; //---------------------------------------------------------------- diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp index dd4109da5cf..824688a75f6 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp @@ -56,6 +56,8 @@ SearchAdapter::createReply() r.coverage = SearchReply::Coverage(_searchInfo->_activeDocs, _searchInfo->_coverageDocs); r.coverage.setSoonActive(_searchInfo->_soonActiveDocs); r.coverage.setDegradeReason(_searchInfo->_degradeReason); + r.coverage.setNodesQueried(_searchInfo->_nodesQueried); + r.coverage.setNodesReplied(_searchInfo->_nodesReplied); FastS_hitresult *hitbuf = _queryResult->_hitbuf; r.hits.resize(hitcnt); diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h index d60b39c52be..de941505fc6 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h +++ b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h @@ -35,7 +35,7 @@ private: void writeLog(); void cleanup(); - virtual void Run(FastOS_ThreadInterface *, void *) override; + void Run(FastOS_ThreadInterface *, void *) override; public: SearchAdapter(FastS_AppContext *appCtx, diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp index 85599b9e897..98642c6edf6 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp @@ -884,7 +884,7 @@ FastS_FNET_Search::CheckCoverage() if ((cntNone > 0) && (cntNone != _nodes.size())) { activeDocs += cntNone * activeDocs/(_nodes.size() - cntNone); } - _util.SetCoverage(covDocs, activeDocs, soonActiveDocs, degradedReason); + _util.SetCoverage(covDocs, activeDocs, soonActiveDocs, degradedReason, _nodes.size(), _nodes.size() - cntNone); } diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h index 6fdca46530c..442fa3f0710 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h @@ -80,12 +80,15 @@ public: ret = 0.0; return ret; } - void SetCoverage(uint64_t coverageDocs, uint64_t activeDocs, uint64_t soonActiveDocs, uint32_t degradeReason) + void SetCoverage(uint64_t coverageDocs, uint64_t activeDocs, uint64_t soonActiveDocs, + uint32_t degradeReason, uint16_t nodesQueried, uint16_t nodesReplied) { _searchInfo._coverageDocs = coverageDocs; _searchInfo._activeDocs = activeDocs; _searchInfo._soonActiveDocs = soonActiveDocs; _searchInfo._degradeReason = degradeReason; + _searchInfo._nodesQueried = nodesQueried; + _searchInfo._nodesReplied = nodesReplied; } void SetAlignedHitCount(uint32_t alignedHitCount) { if (alignedHitCount > _alignedMaxHits) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h index 67e52cbe8ee..e6fce89e82d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h @@ -10,8 +10,7 @@ #include <algorithm> #include <vector> -namespace proton { -namespace matching { +namespace proton::matching { /** * A range of document ids representing a subset of the search space. @@ -205,5 +204,4 @@ public: DocidRange share_range(size_t, DocidRange todo) override; }; -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index bd6a06ac4f1..c1c4387cf41 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -140,7 +140,7 @@ MatchThread::try_share(DocidRange &docid_range, uint32_t next_docid) { } template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> -bool +uint32_t MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) { SearchIterator *search = &tools.search(); @@ -164,7 +164,7 @@ MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange do docId = Strategy::seek_next(*search, docId + 1); } } - return (docId < docid_range.end); + return docId; } template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> @@ -172,12 +172,15 @@ void MatchThread::match_loop(MatchTools &tools, HitCollector &hits) { bool softDoomed = false; + uint32_t docsCovered = 0; Context context(matchParams.rankDropLimit, tools, hits, num_threads); for (DocidRange docid_range = scheduler.first_range(thread_id); !docid_range.empty() && ! softDoomed; docid_range = scheduler.next_range(thread_id)) { - softDoomed = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range); + uint32_t lastCovered = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range); + softDoomed = (lastCovered < docid_range.end); + docsCovered += std::min(lastCovered, docid_range.end) - docid_range.begin; } uint32_t matches = context.matches; if (do_limit && context.isBelowLimit()) { @@ -187,6 +190,7 @@ MatchThread::match_loop(MatchTools &tools, HitCollector &hits) estimate_match_frequency(matches, searchedSoFar); tools.match_limiter().updateDocIdSpaceEstimate(searchedSoFar, 0); } + thread_stats.docsCovered(docsCovered); thread_stats.docsMatched(matches); thread_stats.softDoomed(softDoomed); if (do_rank) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index b08323fa099..b7ecf149001 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -75,7 +75,7 @@ private: bool try_share(DocidRange &docid_range, uint32_t next_docid) __attribute__((noinline)); template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> - bool inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) __attribute__((noinline)); + uint32_t inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) __attribute__((noinline)); template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> void match_loop(MatchTools &tools, HitCollector &hits) __attribute__((noinline)); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index bbe6f604e4e..f21293299c7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -14,7 +14,6 @@ #include <vespa/searchlib/engine/searchreply.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/plugin/setup.h> -#include <cmath> #include <vespa/log/log.h> LOG_SETUP(".proton.matching.matcher"); @@ -92,11 +91,8 @@ bool willNotNeedRanking(const SearchRequest & request, const GroupingContext & g } // namespace proton::matching::<unnamed> FeatureSet::SP -Matcher::getFeatureSet(const DocsumRequest & req, - ISearchContext & searchCtx, - IAttributeContext & attrCtx, - SessionManager & sessionMgr, - bool summaryFeatures) +Matcher::getFeatureSet(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, + SessionManager & sessionMgr, bool summaryFeatures) { SessionId sessionId(&req.sessionId[0], req.sessionId.size()); if (!sessionId.empty()) { @@ -114,22 +110,18 @@ Matcher::getFeatureSet(const DocsumRequest & req, } StupidMetaStore metaStore; - MatchToolsFactory::UP mtf = create_match_tools_factory(req, searchCtx, attrCtx, metaStore, req.propertiesMap.featureOverrides()); + MatchToolsFactory::UP mtf = create_match_tools_factory(req, searchCtx, attrCtx, metaStore, + req.propertiesMap.featureOverrides()); if (!mtf->valid()) { - LOG(warning, "getFeatureSet(%s): query execution failed " - "(invalid query). Returning empty feature set", - (summaryFeatures ? "summary features" : "rank features")); + LOG(warning, "getFeatureSet(%s): query execution failed (invalid query). Returning empty feature set", + (summaryFeatures ? "summary features" : "rank features")); return FeatureSet::SP(new FeatureSet()); } return findFeatureSet(req, *mtf, summaryFeatures); } -Matcher::Matcher(const search::index::Schema &schema, - const Properties &props, - const vespalib::Clock &clock, - QueryLimiter &queryLimiter, - const IConstantValueRepo &constantValueRepo, - uint32_t distributionKey) +Matcher::Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, + QueryLimiter &queryLimiter, const IConstantValueRepo &constantValueRepo, uint32_t distributionKey) : _indexEnv(schema, props, constantValueRepo), _blueprintFactory(), _rankSetup(), @@ -163,21 +155,21 @@ using search::fef::indexproperties::softtimeout::Enabled; using search::fef::indexproperties::softtimeout::Factor; std::unique_ptr<MatchToolsFactory> -Matcher::create_match_tools_factory(const search::engine::Request &request, - ISearchContext &searchContext, - IAttributeContext &attrContext, - const search::IDocumentMetaStore &metaStore, +Matcher::create_match_tools_factory(const search::engine::Request &request, ISearchContext &searchContext, + IAttributeContext &attrContext, const search::IDocumentMetaStore &metaStore, const Properties &feature_overrides) const { const Properties & rankProperties = request.propertiesMap.rankProperties(); bool softTimeoutEnabled = Enabled::lookup(rankProperties, _rankSetup->getSoftTimeoutEnabled()); - double factor = 0.95; + double factor = softTimeoutEnabled + ? Factor::lookup(rankProperties, _stats.softDoomFactor()) + : 0.95; + int64_t safeLeft = request.getTimeLeft() * factor; + fastos::TimeStamp safeDoom(fastos::ClockSystem::now() + safeLeft); if (softTimeoutEnabled) { - factor = Factor::lookup(rankProperties, _stats.softDoomFactor()); - LOG(debug, "Enabling soft-timeout computed factor=%1.3f, used factor=%1.3f", _stats.softDoomFactor(), factor); + LOG(debug, "Soft-timeout computed factor=%1.3f, used factor=%1.3f, softTimeout=%lu softDoom=%ld hardDoom=%ld", + _stats.softDoomFactor(), factor, safeLeft, safeDoom.ns(), request.getTimeOfDoom().ns()); } - uint64_t safeLeft = request.getTimeLeft() * factor; - fastos::TimeStamp safeDoom(fastos::ClockSystem::now() + safeLeft); return std::make_unique<MatchToolsFactory>(_queryLimiter, vespalib::Doom(_clock, safeDoom), vespalib::Doom(_clock, request.getTimeOfDoom()), searchContext, attrContext, request.getStackRef(), request.location, _viewResolver, @@ -185,8 +177,7 @@ Matcher::create_match_tools_factory(const search::engine::Request &request, } SearchReply::UP -Matcher::handleGroupingSession(SessionManager &sessionMgr, - GroupingContext & groupingContext, +Matcher::handleGroupingSession(SessionManager &sessionMgr, GroupingContext & groupingContext, GroupingSession::UP groupingSession) { SearchReply::UP reply = std::make_unique<SearchReply>(); @@ -209,12 +200,9 @@ Matcher::computeNumThreadsPerSearch(Blueprint::HitEstimate hits, const Propertie } SearchReply::UP -Matcher::match(const SearchRequest &request, - vespalib::ThreadBundle &threadBundle, - ISearchContext &searchContext, - IAttributeContext &attrContext, - SessionManager &sessionMgr, - const search::IDocumentMetaStore &metaStore, +Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundle, + ISearchContext &searchContext, IAttributeContext &attrContext, + SessionManager &sessionMgr, const search::IDocumentMetaStore &metaStore, SearchSession::OwnershipBundle &&owned_objects) { fastos::StopWatch total_matching_time; @@ -279,15 +267,7 @@ Matcher::match(const SearchRequest &request, sessionMgr.insert(std::move(session)); } reply = std::move(result->_reply); - SearchReply::Coverage & coverage = reply->coverage; - if (wasLimited) { - LOG(debug, "was limited, degraded from match phase"); - coverage.degradeMatchPhase(); - } - if (my_stats.softDoomed()) { - LOG(debug, "soft doomed, degraded from timeout"); - coverage.degradeTimeout(); - } + uint32_t numActiveLids = metaStore.getNumActiveLids(); // note: this is actually totalSpace+1, since 0 is reserved uint32_t totalSpace = metaStore.getCommittedDocIdLimit(); @@ -303,10 +283,20 @@ Matcher::match(const SearchRequest &request, } size_t covered = (spaceEstimate * numActiveLids) / totalSpace; LOG(debug, "covered = %zd", covered); + + SearchReply::Coverage & coverage = reply->coverage; coverage.setActive(numActiveLids); //TODO this should be calculated with ClusterState calculator. coverage.setSoonActive(numActiveLids); coverage.setCovered(covered); + if (wasLimited) { + coverage.degradeMatchPhase(); + LOG(debug, "was limited, degraded from match phase"); + } + if (my_stats.softDoomed()) { + coverage.degradeTimeout(); + coverage.setCovered(my_stats.docsCovered()); + LOG(debug, "soft doomed, degraded from timeout covered = %lu", coverage.getCovered()); } LOG(debug, "numThreadsPerSearch = %zu. Configured = %d, estimated hits=%d, totalHits=%ld", numThreadsPerSearch, _rankSetup->getNumThreadsPerSearch(), estHits, reply->totalHitCount); } @@ -326,19 +316,15 @@ Matcher::match(const SearchRequest &request, } FeatureSet::SP -Matcher::getSummaryFeatures(const DocsumRequest & req, - ISearchContext & searchCtx, - IAttributeContext & attrCtx, - SessionManager &sessionMgr) +Matcher::getSummaryFeatures(const DocsumRequest & req, ISearchContext & searchCtx, + IAttributeContext & attrCtx, SessionManager &sessionMgr) { return getFeatureSet(req, searchCtx, attrCtx, sessionMgr, true); } FeatureSet::SP -Matcher::getRankFeatures(const DocsumRequest & req, - ISearchContext & searchCtx, - IAttributeContext & attrCtx, - SessionManager &sessionMgr) +Matcher::getRankFeatures(const DocsumRequest & req, ISearchContext & searchCtx, + IAttributeContext & attrCtx, SessionManager &sessionMgr) { return getFeatureSet(req, searchCtx, attrCtx, sessionMgr, false); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp index de46386d6e1..7ea417bb7d2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp @@ -18,6 +18,7 @@ MatchingStats::Partition &get_writable_partition(std::vector<MatchingStats::Part MatchingStats::MatchingStats() : _queries(0), _limited_queries(0), + _docsCovered(0), _docsMatched(0), _docsRanked(0), _docsReRanked(0), @@ -31,13 +32,14 @@ MatchingStats::MatchingStats() _partitions() { } -MatchingStats::~MatchingStats() { } +MatchingStats::~MatchingStats() = default; MatchingStats & MatchingStats::merge_partition(const Partition &partition, size_t id) { get_writable_partition(_partitions, id) = partition; + _docsCovered += partition.docsCovered(); _docsMatched += partition.docsMatched(); _docsRanked += partition.docsRanked(); _docsReRanked += partition.docsReRanked(); @@ -51,10 +53,10 @@ MatchingStats::merge_partition(const Partition &partition, size_t id) MatchingStats & MatchingStats::add(const MatchingStats &rhs) { - _queries += rhs._queries; _limited_queries += rhs._limited_queries; + _docsCovered += rhs._docsCovered; _docsMatched += rhs._docsMatched; _docsRanked += rhs._docsRanked; _docsReRanked += rhs._docsReRanked; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h index 99084098f6f..e4578547e60 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h @@ -37,6 +37,7 @@ public: * thread. **/ class Partition { + size_t _docsCovered; size_t _docsMatched; size_t _docsRanked; size_t _docsReRanked; @@ -45,13 +46,16 @@ public: Avg _wait_time; public: Partition() - : _docsMatched(0), + : _docsCovered(0), + _docsMatched(0), _docsRanked(0), _docsReRanked(0), _softDoomed(0), _active_time(), _wait_time() { } + Partition &docsCovered(size_t value) { _docsCovered = value; return *this; } + size_t docsCovered() const { return _docsCovered; } Partition &docsMatched(size_t value) { _docsMatched = value; return *this; } size_t docsMatched() const { return _docsMatched; } Partition &docsRanked(size_t value) { _docsRanked = value; return *this; } @@ -69,6 +73,7 @@ public: size_t wait_time_count() const { return _wait_time.count(); } Partition &add(const Partition &rhs) { + _docsCovered += rhs.docsCovered(); _docsMatched += rhs._docsMatched; _docsRanked += rhs._docsRanked; _docsReRanked += rhs._docsReRanked; @@ -83,6 +88,7 @@ public: private: size_t _queries; size_t _limited_queries; + size_t _docsCovered; size_t _docsMatched; size_t _docsRanked; size_t _docsReRanked; @@ -109,6 +115,9 @@ public: MatchingStats &limited_queries(size_t value) { _limited_queries = value; return *this; } size_t limited_queries() const { return _limited_queries; } + MatchingStats &docsCovered(size_t value) { _docsCovered = value; return *this; } + size_t docsCovered() const { return _docsCovered; } + MatchingStats &docsMatched(size_t value) { _docsMatched = value; return *this; } size_t docsMatched() const { return _docsMatched; } diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp index c4da5ca4873..ad4cf02f4e8 100644 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ b/searchlib/src/tests/common/packets/packets_test.cpp @@ -2,7 +2,6 @@ #include <vespa/searchlib/common/mapnames.h> #include <vespa/searchlib/common/packets.h> -#include <vector> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/fnet/controlpacket.h> @@ -10,15 +9,6 @@ using namespace search::fs4transport; using vespalib::compression::CompressionConfig; - -// ---------------------------------------------------------------------------- -// -// Utilities -// -// ---------------------------------------------------------------------------- - -#define QRF_RANKTYPE QRF_RANKTYPE_DOUBLE - #define PCODE_BEGIN PCODE_EOL #define PCODE_END PCODE_LastCode @@ -59,7 +49,7 @@ testEncodeDecode(FS4PersistentPacketStreamer &streamer, FNET_Packet &packet) EXPECT_EQUAL(myStreamer.getChannelId(pcode, 1u), chid); FNET_Packet *ret = streamer.Decode(&buf, plen, pcode, ctx); - ASSERT_TRUE(ret); + assert(ret); if (ret->GetPCODE() == (pcode & PCODE_MASK)) { FNET_DataBuffer rhs; streamer.Encode(ret, 1u, &rhs); @@ -325,6 +315,10 @@ TEST("testQueryResultX") { src->setDistributionKey(4u); src->_coverageDocs = 6u; src->_activeDocs = 7u; + src->_soonActiveDocs = 8; + src->_coverageDegradeReason = 0x17; + src->setNodesQueried(12); + src->setNodesReplied(11); uint32_t sortIndex[3] = { 0u, 1u, 3u /* size of data */}; // numDocs + 1 src->SetSortDataRef(2, sortIndex, "foo"); src->SetGroupDataRef("baz", 3u); @@ -358,8 +352,12 @@ TEST("testQueryResultX") { EXPECT_EQUAL(2u, ptr->_totNumDocs); EXPECT_EQUAL((search::HitRank)3, ptr->_maxRank); EXPECT_EQUAL(4u, ptr->getDistributionKey()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE ? 6u : 0u, ptr->_coverageDocs); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE ? 7u : 0u, ptr->_activeDocs); + EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 12 : 0u, ptr->getNodesQueried()); + EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 11 : 0u, ptr->getNodesReplied()); + EXPECT_EQUAL(6u, ptr->_coverageDocs); + EXPECT_EQUAL(7u, ptr->_activeDocs); + EXPECT_EQUAL(8u, ptr->_soonActiveDocs); + EXPECT_EQUAL(0x17u, ptr->_coverageDegradeReason); if (ptr->_features & QRF_SORTDATA) { EXPECT_EQUAL(0u, ptr->_sortIndex[0]); EXPECT_EQUAL(1u, ptr->_sortIndex[1]); diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 281d0c1541e..04da47aa76b 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -906,6 +906,8 @@ FS4Packet_QUERYRESULTX::AllocateHits(uint32_t cnt) FS4Packet_QUERYRESULTX::FS4Packet_QUERYRESULTX() : FS4Packet(), _distributionKey(0), + _nodesQueried(0), + _nodesReplied(0), _features(0), _offset(0), _numDocs(0), @@ -942,22 +944,14 @@ FS4Packet_QUERYRESULTX::GetLength() _numDocs * (sizeof(document::GlobalId) + sizeof(search::HitRank)); plen += sizeof(uint32_t); - - if ((_features & QRF_MLD) != 0) - plen += _numDocs * 2 * sizeof(uint32_t); + plen += (_features & QRF_COVERAGE_NODES) ? 2 * sizeof(uint16_t) : 0; + plen += (_features & QRF_MLD) ? _numDocs * 2 * sizeof(uint32_t) : 0; + plen += (_features & QRF_GROUPDATA) ? sizeof(uint32_t) + _groupDataLen : 0; + plen += 3 * sizeof(uint64_t) + sizeof(uint32_t); if (((_features & QRF_SORTDATA) != 0) && (_numDocs > 0)) plen += _numDocs * sizeof(uint32_t) + (_sortIndex[_numDocs] - _sortIndex[0]); - if ((_features & QRF_GROUPDATA) != 0) - plen += sizeof(uint32_t) + _groupDataLen; - - if ((_features & QRF_COVERAGE) != 0) - plen += 2 * sizeof(uint64_t); - - if ((_features & QRF_EXTENDED_COVERAGE) != 0) - plen += sizeof(uint64_t) + sizeof(uint32_t); - if ((_features & QRF_PROPERTIES) != 0) { plen += sizeof(uint32_t); for (uint32_t i = 0; i < _propsVector.size(); ++i) { @@ -981,6 +975,11 @@ FS4Packet_QUERYRESULTX::Encode(FNET_DataBuffer *dst) dst->WriteInt64Fast(mrval.INT64); dst->WriteInt32Fast(_distributionKey); + if (_features & QRF_COVERAGE_NODES) { + dst->WriteInt16Fast(_nodesQueried); + dst->WriteInt16Fast(_nodesReplied); + } + if (((_features & QRF_SORTDATA) != 0) && (_numDocs > 0)) { @@ -998,14 +997,11 @@ FS4Packet_QUERYRESULTX::Encode(FNET_DataBuffer *dst) dst->WriteBytesFast(_groupData, _groupDataLen); } - if ((_features & QRF_COVERAGE) != 0) { - dst->WriteInt64Fast(_coverageDocs); - dst->WriteInt64Fast(_activeDocs); - } - if ((_features & QRF_EXTENDED_COVERAGE) != 0) { - dst->WriteInt64Fast(_soonActiveDocs); - dst->WriteInt32Fast(_coverageDegradeReason); - } + dst->WriteInt64Fast(_coverageDocs); + dst->WriteInt64Fast(_activeDocs); + dst->WriteInt64Fast(_soonActiveDocs); + dst->WriteInt32Fast(_coverageDegradeReason); + for (uint32_t i = 0; i < _numDocs; i++) { dst->WriteBytesFast(_hits[i]._gid.get(), document::GlobalId::LENGTH); @@ -1051,8 +1047,13 @@ FS4Packet_QUERYRESULTX::Decode(FNET_DataBuffer *src, uint32_t len) _distributionKey = src->ReadInt32(); len -= 3 * sizeof(uint32_t) + sizeof(uint64_t) + sizeof(search::HitRank); - if (((_features & QRF_SORTDATA) != 0) && - (_numDocs > 0)) { + if (_features & QRF_COVERAGE_NODES) { + if (len < 2* sizeof(uint16_t)) goto error; + _nodesQueried = src->ReadInt16(); + _nodesReplied = src->ReadInt16(); + len -= 2*sizeof(uint16_t); + } + if (((_features & QRF_SORTDATA) != 0) && (_numDocs > 0)) { if (len < _numDocs * sizeof(uint32_t)) goto error; AllocateSortIndex(_numDocs); _sortIndex[0] = 0; // implicit @@ -1079,19 +1080,13 @@ FS4Packet_QUERYRESULTX::Decode(FNET_DataBuffer *src, uint32_t len) len -= _groupDataLen; } - if ((_features & QRF_COVERAGE) != 0) { - if (len < 2 * sizeof(uint64_t)) goto error; - _coverageDocs = src->ReadInt64(); - _activeDocs = src->ReadInt64(); - len -= 2 * sizeof(uint64_t); - } - if ((_features & QRF_EXTENDED_COVERAGE) != 0) { - if (len < sizeof(uint64_t) + sizeof(uint32_t)) goto error; - _soonActiveDocs = src->ReadInt64(); - _coverageDegradeReason = src->ReadInt32(); + if (len < 3 * sizeof(uint64_t) + sizeof(uint32_t)) goto error; + _coverageDocs = src->ReadInt64(); + _activeDocs = src->ReadInt64(); + _soonActiveDocs = src->ReadInt64(); + _coverageDegradeReason = src->ReadInt32(); - len -= sizeof(uint64_t) + sizeof(uint32_t); - } + len -= 3*sizeof(uint64_t) + sizeof(uint32_t); if ((_features & QRF_MLD) != 0) { hitSize += 2 * sizeof(uint32_t); diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index 52130c57374..a876400e503 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -21,6 +21,7 @@ using vespalib::string; enum fnet_feature_masks { FNET_QRF_SUPPORTED_MASK = (QRF_MLD | QRF_SORTDATA | + QRF_COVERAGE_NODES | QRF_EXTENDED_COVERAGE | QRF_COVERAGE | QRF_GROUPDATA | @@ -65,8 +66,7 @@ private: uint32_t _used; public: - PacketArray(FNET_Packet **arr = NULL, - uint32_t size = 0) + PacketArray(FNET_Packet **arr = nullptr, uint32_t size = 0) : _extArray(arr), _array(arr), _size(size), @@ -374,6 +374,8 @@ private: FS4Packet_QUERYRESULTX& operator=(const FS4Packet_QUERYRESULTX &); uint32_t _distributionKey; + uint16_t _nodesQueried; + uint16_t _nodesReplied; public: uint32_t _features; // see queryresult_features @@ -425,8 +427,11 @@ public: vespalib::string toString(uint32_t indent) const override ; uint32_t getDistributionKey() const { return _distributionKey; } void setDistributionKey(uint32_t key) { _distributionKey = key; } + uint16_t getNodesQueried() const { return _nodesQueried; } + void setNodesQueried(uint16_t key) { _nodesQueried = key; } + uint16_t getNodesReplied() const { return _nodesReplied; } + void setNodesReplied(uint16_t key) { _nodesReplied = key; } }; - //========================================================================== class FS4Packet_QUERYX : public FS4Packet diff --git a/searchlib/src/vespa/searchlib/common/transport.h b/searchlib/src/vespa/searchlib/common/transport.h index 454c8404163..b6a925240c9 100644 --- a/searchlib/src/vespa/searchlib/common/transport.h +++ b/searchlib/src/vespa/searchlib/common/transport.h @@ -25,6 +25,7 @@ namespace search::fs4transport { **/ enum queryflags { QFLAG_EXTENDED_COVERAGE = 0x00000001, + QFLAG_COVERAGE_NODES = 0x00000002, QFLAG_ESTIMATE = 0x00000080, QFLAG_DROP_SORTDATA = 0x00004000, QFLAG_NO_RESULTCACHE = 0x00010000, @@ -46,6 +47,7 @@ enum queryflags { **/ enum queryresult_features { QRF_MLD = 0x00000001, + QRF_COVERAGE_NODES = 0x00000002, QRF_SORTDATA = 0x00000010, QRF_EXTENDED_COVERAGE = 0x00000020, QRF_COVERAGE = 0x00000040, diff --git a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp index 0697c4631ba..5b7b6ef8265 100644 --- a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp +++ b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp @@ -117,13 +117,15 @@ PacketConverter::fromSearchReply(const SearchReply &reply, QUERYRESULTX &packet) packet.AllocateGroupData(reply.groupResult.size()); memcpy(packet._groupData, &(reply.groupResult[0]), reply.groupResult.size()); } - packet._features |= QRF_COVERAGE; + packet._features |= QRF_COVERAGE | QRF_EXTENDED_COVERAGE; packet._coverageDocs = reply.coverage.getCovered(); packet._activeDocs = reply.coverage.getActive(); packet._soonActiveDocs = reply.coverage.getSoonActive(); packet._coverageDegradeReason = reply.coverage.getDegradeReason(); - if (reply.request && (reply.request->queryFlags & QFLAG_EXTENDED_COVERAGE)) { - packet._features |= QRF_EXTENDED_COVERAGE; + packet.setNodesQueried(reply.coverage.getNodesQueried()); + packet.setNodesReplied(reply.coverage.getNodesReplied()); + if (reply.request && (reply.request->queryFlags & QFLAG_COVERAGE_NODES)) { + packet._features |= QRF_COVERAGE_NODES; } if (reply.useWideHits) { packet._features |= QRF_MLD; diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index b1c55fea874..9135866a7f7 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -32,15 +32,24 @@ public: public: Coverage() : Coverage(0) { } Coverage(uint64_t active) : Coverage(active, active) { } - Coverage(uint64_t active, uint64_t covered) : _covered(covered), _active(active), _soonActive(active), _degradeReason(0) { } + Coverage(uint64_t active, uint64_t covered) + : _covered(covered), _active(active), _soonActive(active), + _degradeReason(0), _nodesQueried(1), _nodesReplied(1) + { } uint64_t getCovered() const { return _covered; } uint64_t getActive() const { return _active; } uint64_t getSoonActive() const { return _soonActive; } uint32_t getDegradeReason() const { return _degradeReason; } + uint16_t getNodesQueried() const { return _nodesQueried; } + uint16_t getNodesReplied() const { return _nodesReplied; } + Coverage & setCovered(uint64_t v) { _covered = v; return *this; } Coverage & setActive(uint64_t v) { _active = v; return *this; } Coverage & setSoonActive(uint64_t v) { _soonActive = v; return *this; } Coverage & setDegradeReason(uint32_t v) { _degradeReason = v; return *this; } + Coverage & setNodesQueried(uint16_t v) { _nodesQueried = v; return *this; } + Coverage & setNodesReplied(uint16_t v) { _nodesReplied = v; return *this; } + Coverage & degradeMatchPhase() { _degradeReason |= MATCH_PHASE; return *this; } Coverage & degradeTimeout() { _degradeReason |= TIMEOUT; return *this; } Coverage & degradeAdaptiveTimeout() { _degradeReason |= ADAPTIVE_TIMEOUT; return *this; } @@ -50,6 +59,8 @@ public: uint64_t _active; uint64_t _soonActive; uint32_t _degradeReason; + uint16_t _nodesQueried; + uint16_t _nodesReplied; }; // set to false to indicate 'talk to the hand' behavior |