Handle sort data

5 files changed, 72 insertions, 63 deletions

diff --git a/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java b/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java
index f9fce278d9e..85e1aef3da0 100644
--- a/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java
+++ b/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java
@@ -28,10 +28,8 @@ public abstract class BasicPacket {
 
     protected ByteBuffer encodingBuffer;
 
-    /**
-     * The length of this packet in bytes or -1 if not known
-     */
-    protected int length=-1;
+    /** The length of this packet in bytes or -1 if not known */
+    protected int length = -1;
 
     /**
      * A timestamp which can be set or inspected by clients of this class
@@ -49,6 +47,7 @@ public abstract class BasicPacket {
     /**
      * Sets the number of bytes the package must be before activating compression.
      * A value of 0 means no compression.
+     *
      * @param limit smallest package size that triggers compression.
      */
     public void setCompressionLimit(int limit) { compressionLimit = limit; }
@@ -64,8 +63,8 @@ public abstract class BasicPacket {
      * @throws UnsupportedOperationException if not implemented in the subclass
      */
     public BasicPacket decode(ByteBuffer buffer) {
-        length=buffer.getInt()+4; // Streamed packet length is the length-4
-        int code=buffer.getInt();
+        length = buffer.getInt()+4; // Streamed packet length is the length-4
+        int code = buffer.getInt();
 
         decodeAndDecompressBody(buffer, code, length - 2*4);
         return this;
@@ -88,7 +87,8 @@ public abstract class BasicPacket {
                 compressedData = new byte[compressedSize];
                 buffer.get(compressedData);
             }
-            byte[] body = compressor.decompress(CompressionType.valueOf(compressionType), compressedData, offset, uncompressedSize, Optional.of(compressedSize));
+            byte[] body = compressor.decompress(CompressionType.valueOf(compressionType), compressedData, offset,
+                                                uncompressedSize, Optional.of(compressedSize));
             ByteBuffer bodyBuffer = ByteBuffer.wrap(body);
             length += uncompressedSize - (compressedSize + 4);
             decodeBody(bodyBuffer);
@@ -104,8 +104,7 @@ public abstract class BasicPacket {
      * @throws UnsupportedOperationException if not implemented in the subclass
      */
     public void decodeBody(ByteBuffer buffer) {
-        throw new UnsupportedOperationException("Decoding of " + this +
-                                                " is not implemented");
+        throw new UnsupportedOperationException("Decoding of " + this + " is not implemented");
     }
 
     /**
@@ -115,7 +114,7 @@ public abstract class BasicPacket {
      * will use this method to store the code.
      */
     protected void codeDecodedHook(int code) {
-        if (code!=getCode())
+        if (code != getCode())
             throw new RuntimeException("Can not decode " + code + " into " + this);
     }
 
@@ -132,9 +131,7 @@ public abstract class BasicPacket {
      * @return this for convenience
      * @throws UnsupportedOperationException if not implemented in the subclass
      */
-    public BasicPacket encode(ByteBuffer buffer)
-        throws BufferTooSmallException
-    {
+    public BasicPacket encode(ByteBuffer buffer) throws BufferTooSmallException {
         int oldLimit = buffer.limit();
         int startPosition = buffer.position();
 
@@ -227,7 +224,7 @@ public abstract class BasicPacket {
         allocateAndEncode(channelId, DEFAULT_WRITE_BUFFER_SIZE);
     }
 
-    private final void allocateAndEncode(int channelId, int initialSize) {
+    private void allocateAndEncode(int channelId, int initialSize) {
         if (encodingBuffer != null) {
             patchChannelId(encodingBuffer, channelId);
             return;
@@ -261,25 +258,23 @@ public abstract class BasicPacket {
      * remove internal reference to it.
      */
     public final ByteBuffer grantEncodingBuffer(int channelId) {
-        ByteBuffer b;
         if (encodingBuffer == null) {
             allocateAndEncode(channelId);
         } else {
             patchChannelId(encodingBuffer, channelId);
         }
-        b = encodingBuffer;
+        ByteBuffer b = encodingBuffer;
         encodingBuffer = null;
         return b;
     }
 
     public final ByteBuffer grantEncodingBuffer(int channelId, int initialSize) {
-        ByteBuffer b;
         if (encodingBuffer == null) {
             allocateAndEncode(channelId, initialSize);
         } else {
             patchChannelId(encodingBuffer, channelId);
         }
-        b = encodingBuffer;
+        ByteBuffer b = encodingBuffer;
         encodingBuffer = null;
         return b;
     }
diff --git a/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java b/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java
index 21e9464129e..ad679e0c53c 100644
--- a/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java
+++ b/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java
@@ -52,8 +52,7 @@ public class DocsumPacket extends Packet {
 
     public String toString() {
         return "docsum packet [globalId: " + globalId.toString() +
-            ", size: " + (data==null ? "(no data)" : data.length +
-                          " bytes") + " ]";
+               ", size: " + (data==null ? "(no data)" : data.length + " bytes") + " ]";
     }
 
 }
diff --git a/container-search/src/main/java/com/yahoo/fs4/Packet.java b/container-search/src/main/java/com/yahoo/fs4/Packet.java
index 6ea5ce9bc6d..78d5083c25f 100644
--- a/container-search/src/main/java/com/yahoo/fs4/Packet.java
+++ b/container-search/src/main/java/com/yahoo/fs4/Packet.java
@@ -10,14 +10,16 @@ import java.util.logging.Logger;
  * @author bratseth
  */
 public abstract class Packet extends BasicPacket {
+
     private static Logger log = Logger.getLogger(Packet.class.getName());
+
     /**
      * The channel at which this packet will be sent or was received,
      * or -1 when this is not known
      */
-    protected int channel=-1;
+    protected int channel = -1;
 
-    protected static final int CHANNEL_ID_OFFSET = 8;
+    private static final int CHANNEL_ID_OFFSET = 8;
 
     /**
      * Fills this package from a byte buffer positioned at the first
@@ -28,21 +30,18 @@ public abstract class Packet extends BasicPacket {
      */
     public BasicPacket decode(ByteBuffer buffer) {
         int originalPos = buffer.position();
-        length=buffer.getInt()+4; // Streamed packet length is the length-4
+        length = buffer.getInt()+4; // Streamed packet length is the length-4
         int packetLength = length;
         try {
-            int code=buffer.getInt();
-            channel=buffer.getInt();
+            int code = buffer.getInt();
+            channel = buffer.getInt();
 
             decodeAndDecompressBody(buffer, code, length - 3*4);
         }
         finally {
             int targetPosition = (originalPos + packetLength);
             if (buffer.position() != targetPosition) {
-                log.warning(" position in buffer, is "
-                            + buffer.position()
-                            + " should be "
-                            + targetPosition);
+                log.warning("Position in buffer is " + buffer.position() + " should be " + targetPosition);
                 buffer.position(targetPosition);
             }
         }
@@ -64,7 +63,7 @@ public abstract class Packet extends BasicPacket {
      * @throws UnsupportedOperationException if not implemented in the subclass
      */
     public final Packet encode(ByteBuffer buffer, int channel) throws BufferTooSmallException {
-        this.channel=channel;
+        this.channel = channel;
         int oldLimit = buffer.limit();
         int startPosition = buffer.position();
 
diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java
index 45a06f6b4f3..b46b96bfbd6 100644
--- a/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java
+++ b/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java
@@ -9,19 +9,20 @@ import java.util.List;
 
 
 /**
- * An "extended query result" packet. This is the query result packets used today,
- * they allow more flexible sets of parameters to be shipped with query results.
- * This packet can be decoded only.
+ * A query result packet (code 217). This packet can be decoded only.
  *
  * @author bratseth
  */
 public class QueryResultPacket extends Packet {
 
-    /** This may have code 202, 208, 214 or 217 of historical reasons */
-    private int code;
+    /** The code of this type of package */
+    private static final int code = 217;
 
-    /** Whether mld stuff, whatever that is, is included in this result */
-    private boolean mldFeature=false;
+    /** Whether mld data is included in this result */
+    private boolean mldFeature = false;
+
+    /** Whether sort data is included in this result */
+    private boolean sortData = false;
 
     /** Whether coverage information is included in this result */
     private boolean coverageNodes = false;
@@ -90,21 +91,29 @@ public class QueryResultPacket extends Packet {
 
     @Override
     public void decodeBody(ByteBuffer buffer) {
-        IntBuffer ints=buffer.asIntBuffer();
-
+        IntBuffer ints = buffer.asIntBuffer();
         decodeFeatures(ints);
         offset = ints.get();
-        int documentCount=ints.get();
-        buffer.position(buffer.position() + ints.position()*4);
+        int documentCount = ints.get();
+        buffer.position(buffer.position() + ints.position() * 4);
         totalDocumentCount = buffer.getLong();
         maxRank = decodeMaxRank(buffer);
         ints = buffer.asIntBuffer();
-        docstamp=ints.get();
-        buffer.position(buffer.position() + ints.position()*4);
+        docstamp = ints.get();
+        buffer.position(buffer.position() + ints.position() * 4);
+        // do not access "ints" below here!
+
         if (coverageNodes) {
             nodesQueried = buffer.getShort();
             nodesReplied = buffer.getShort();
         }
+
+        if (sortData && documentCount > 0) { // sort data is not needed - skip
+            buffer.position(buffer.position() + (documentCount -1) * 4); // one sortIndex int per document
+            int sortDataLengthInBytes = buffer.getInt();
+            buffer.position(buffer.position() + sortDataLengthInBytes);
+        }
+
         if (groupDataFeature) {
             int len = buffer.getInt();
             groupData = new byte[len];
@@ -142,36 +151,28 @@ public class QueryResultPacket extends Packet {
     public static final int QRF_GROUPDATA       = 0x00000200;
     public static final int QRF_PROPERTIES      = 0x00000400;
 
-    /**
-     * Sets the features of this package.
-     * Features are either encoded by different package codes
-     * or by a feature int, for reasons not easily comprehended.
-     */
+    /** Decodes the feature int of this package data into boolean feature fields */
     private void decodeFeatures(IntBuffer buffer) {
-        switch (getCode()) {
-        case 217:
-                int features=buffer.get();
-                mldFeature       = (QRF_MLD & features) != 0;
-                // Data given by sortFeature not currently used by QRS:
-                // sortFeature   = (QRF_SORTDATA & features) != 0;
-                coverageNodes    = (QRF_COVERAGE_NODES & features) != 0;
-                groupDataFeature = (QRF_GROUPDATA & features) != 0;
-                propsFeature     = (QRF_PROPERTIES & features) != 0;
-                break;
-            default:
-                throw new RuntimeException("Programming error, packet " + getCode() + "Not expected.");
-        }
+        int features=buffer.get();
+        mldFeature       = (QRF_MLD & features) != 0;
+        sortData = (QRF_SORTDATA & features) != 0;
+        coverageNodes    = (QRF_COVERAGE_NODES & features) != 0;
+        groupDataFeature = (QRF_GROUPDATA & features) != 0;
+        propsFeature     = (QRF_PROPERTIES & features) != 0;
     }
 
     private void decodeDocuments(ByteBuffer buffer, int documentCount) {
-        for (int i=0; i<documentCount; i++) {
+        for (int i = 0; i < documentCount; i++) {
             documents.add(new DocumentInfo(buffer, this));
         }
     }
 
     public int getCode() { return code; }
 
-    protected void codeDecodedHook(int code) { this.code=code; }
+    protected void codeDecodedHook(int code) {
+        if ( code != this.code)
+            throw new RuntimeException("Programming error, packet " + getCode() + "Not expected.");
+    }
 
     public int getDocumentCount() { return documents.size(); }
 
diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
index 4e8f516c9a1..369bc17a90c 100644
--- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
+++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
@@ -21,6 +21,7 @@ import com.yahoo.container.jdisc.VespaHeaders;
 import com.yahoo.container.logging.AccessLog;
 import com.yahoo.container.protect.FreezeDetector;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.Response;
 import com.yahoo.language.Linguistics;
 import com.yahoo.log.LogLevel;
 import com.yahoo.net.HostName;
@@ -55,6 +56,8 @@ import com.yahoo.statistics.Value;
 import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
 import edu.umd.cs.findbugs.annotations.NonNull;
 
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -117,6 +120,18 @@ public class SearchHandler extends LoggingRequestHandler {
         }
     }
 
+    private void warmupQueries() {
+        String[] requestUris = new String[] {"warmupRequestUrl1", "warmupRequestUrl2"};
+        int warmupIterations = 50;
+
+        for (int i = 0; i < warmupIterations; i++) {
+            for (String requestUri : requestUris) {
+                handle(HttpRequest.createTestRequest(requestUri, com.yahoo.jdisc.http.HttpRequest.Method.GET));
+            }
+        }
+    }
+
+
     @Inject
     public SearchHandler(
             final ChainsConfig chainsConfig,