From a5af385446b25819f68550d7a2a5ae0570109b7c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 11 Apr 2018 13:51:47 +0200 Subject: Handle sort data --- .../src/main/java/com/yahoo/fs4/BasicPacket.java | 31 +++++----- .../src/main/java/com/yahoo/fs4/DocsumPacket.java | 3 +- .../src/main/java/com/yahoo/fs4/Packet.java | 19 +++--- .../main/java/com/yahoo/fs4/QueryResultPacket.java | 67 +++++++++++----------- .../com/yahoo/search/handler/SearchHandler.java | 15 +++++ 5 files changed, 72 insertions(+), 63 deletions(-) (limited to 'container-search') diff --git a/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java b/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java index f9fce278d9e..85e1aef3da0 100644 --- a/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/BasicPacket.java @@ -28,10 +28,8 @@ public abstract class BasicPacket { protected ByteBuffer encodingBuffer; - /** - * The length of this packet in bytes or -1 if not known - */ - protected int length=-1; + /** The length of this packet in bytes or -1 if not known */ + protected int length = -1; /** * A timestamp which can be set or inspected by clients of this class @@ -49,6 +47,7 @@ public abstract class BasicPacket { /** * Sets the number of bytes the package must be before activating compression. * A value of 0 means no compression. + * * @param limit smallest package size that triggers compression. */ public void setCompressionLimit(int limit) { compressionLimit = limit; } @@ -64,8 +63,8 @@ public abstract class BasicPacket { * @throws UnsupportedOperationException if not implemented in the subclass */ public BasicPacket decode(ByteBuffer buffer) { - length=buffer.getInt()+4; // Streamed packet length is the length-4 - int code=buffer.getInt(); + length = buffer.getInt()+4; // Streamed packet length is the length-4 + int code = buffer.getInt(); decodeAndDecompressBody(buffer, code, length - 2*4); return this; @@ -88,7 +87,8 @@ public abstract class BasicPacket { compressedData = new byte[compressedSize]; buffer.get(compressedData); } - byte[] body = compressor.decompress(CompressionType.valueOf(compressionType), compressedData, offset, uncompressedSize, Optional.of(compressedSize)); + byte[] body = compressor.decompress(CompressionType.valueOf(compressionType), compressedData, offset, + uncompressedSize, Optional.of(compressedSize)); ByteBuffer bodyBuffer = ByteBuffer.wrap(body); length += uncompressedSize - (compressedSize + 4); decodeBody(bodyBuffer); @@ -104,8 +104,7 @@ public abstract class BasicPacket { * @throws UnsupportedOperationException if not implemented in the subclass */ public void decodeBody(ByteBuffer buffer) { - throw new UnsupportedOperationException("Decoding of " + this + - " is not implemented"); + throw new UnsupportedOperationException("Decoding of " + this + " is not implemented"); } /** @@ -115,7 +114,7 @@ public abstract class BasicPacket { * will use this method to store the code. */ protected void codeDecodedHook(int code) { - if (code!=getCode()) + if (code != getCode()) throw new RuntimeException("Can not decode " + code + " into " + this); } @@ -132,9 +131,7 @@ public abstract class BasicPacket { * @return this for convenience * @throws UnsupportedOperationException if not implemented in the subclass */ - public BasicPacket encode(ByteBuffer buffer) - throws BufferTooSmallException - { + public BasicPacket encode(ByteBuffer buffer) throws BufferTooSmallException { int oldLimit = buffer.limit(); int startPosition = buffer.position(); @@ -227,7 +224,7 @@ public abstract class BasicPacket { allocateAndEncode(channelId, DEFAULT_WRITE_BUFFER_SIZE); } - private final void allocateAndEncode(int channelId, int initialSize) { + private void allocateAndEncode(int channelId, int initialSize) { if (encodingBuffer != null) { patchChannelId(encodingBuffer, channelId); return; @@ -261,25 +258,23 @@ public abstract class BasicPacket { * remove internal reference to it. */ public final ByteBuffer grantEncodingBuffer(int channelId) { - ByteBuffer b; if (encodingBuffer == null) { allocateAndEncode(channelId); } else { patchChannelId(encodingBuffer, channelId); } - b = encodingBuffer; + ByteBuffer b = encodingBuffer; encodingBuffer = null; return b; } public final ByteBuffer grantEncodingBuffer(int channelId, int initialSize) { - ByteBuffer b; if (encodingBuffer == null) { allocateAndEncode(channelId, initialSize); } else { patchChannelId(encodingBuffer, channelId); } - b = encodingBuffer; + ByteBuffer b = encodingBuffer; encodingBuffer = null; return b; } diff --git a/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java b/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java index 21e9464129e..ad679e0c53c 100644 --- a/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/DocsumPacket.java @@ -52,8 +52,7 @@ public class DocsumPacket extends Packet { public String toString() { return "docsum packet [globalId: " + globalId.toString() + - ", size: " + (data==null ? "(no data)" : data.length + - " bytes") + " ]"; + ", size: " + (data==null ? "(no data)" : data.length + " bytes") + " ]"; } } diff --git a/container-search/src/main/java/com/yahoo/fs4/Packet.java b/container-search/src/main/java/com/yahoo/fs4/Packet.java index 6ea5ce9bc6d..78d5083c25f 100644 --- a/container-search/src/main/java/com/yahoo/fs4/Packet.java +++ b/container-search/src/main/java/com/yahoo/fs4/Packet.java @@ -10,14 +10,16 @@ import java.util.logging.Logger; * @author bratseth */ public abstract class Packet extends BasicPacket { + private static Logger log = Logger.getLogger(Packet.class.getName()); + /** * The channel at which this packet will be sent or was received, * or -1 when this is not known */ - protected int channel=-1; + protected int channel = -1; - protected static final int CHANNEL_ID_OFFSET = 8; + private static final int CHANNEL_ID_OFFSET = 8; /** * Fills this package from a byte buffer positioned at the first @@ -28,21 +30,18 @@ public abstract class Packet extends BasicPacket { */ public BasicPacket decode(ByteBuffer buffer) { int originalPos = buffer.position(); - length=buffer.getInt()+4; // Streamed packet length is the length-4 + length = buffer.getInt()+4; // Streamed packet length is the length-4 int packetLength = length; try { - int code=buffer.getInt(); - channel=buffer.getInt(); + int code = buffer.getInt(); + channel = buffer.getInt(); decodeAndDecompressBody(buffer, code, length - 3*4); } finally { int targetPosition = (originalPos + packetLength); if (buffer.position() != targetPosition) { - log.warning(" position in buffer, is " - + buffer.position() - + " should be " - + targetPosition); + log.warning("Position in buffer is " + buffer.position() + " should be " + targetPosition); buffer.position(targetPosition); } } @@ -64,7 +63,7 @@ public abstract class Packet extends BasicPacket { * @throws UnsupportedOperationException if not implemented in the subclass */ public final Packet encode(ByteBuffer buffer, int channel) throws BufferTooSmallException { - this.channel=channel; + this.channel = channel; int oldLimit = buffer.limit(); int startPosition = buffer.position(); diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java index 45a06f6b4f3..b46b96bfbd6 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryResultPacket.java @@ -9,19 +9,20 @@ import java.util.List; /** - * An "extended query result" packet. This is the query result packets used today, - * they allow more flexible sets of parameters to be shipped with query results. - * This packet can be decoded only. + * A query result packet (code 217). This packet can be decoded only. * * @author bratseth */ public class QueryResultPacket extends Packet { - /** This may have code 202, 208, 214 or 217 of historical reasons */ - private int code; + /** The code of this type of package */ + private static final int code = 217; - /** Whether mld stuff, whatever that is, is included in this result */ - private boolean mldFeature=false; + /** Whether mld data is included in this result */ + private boolean mldFeature = false; + + /** Whether sort data is included in this result */ + private boolean sortData = false; /** Whether coverage information is included in this result */ private boolean coverageNodes = false; @@ -90,21 +91,29 @@ public class QueryResultPacket extends Packet { @Override public void decodeBody(ByteBuffer buffer) { - IntBuffer ints=buffer.asIntBuffer(); - + IntBuffer ints = buffer.asIntBuffer(); decodeFeatures(ints); offset = ints.get(); - int documentCount=ints.get(); - buffer.position(buffer.position() + ints.position()*4); + int documentCount = ints.get(); + buffer.position(buffer.position() + ints.position() * 4); totalDocumentCount = buffer.getLong(); maxRank = decodeMaxRank(buffer); ints = buffer.asIntBuffer(); - docstamp=ints.get(); - buffer.position(buffer.position() + ints.position()*4); + docstamp = ints.get(); + buffer.position(buffer.position() + ints.position() * 4); + // do not access "ints" below here! + if (coverageNodes) { nodesQueried = buffer.getShort(); nodesReplied = buffer.getShort(); } + + if (sortData && documentCount > 0) { // sort data is not needed - skip + buffer.position(buffer.position() + (documentCount -1) * 4); // one sortIndex int per document + int sortDataLengthInBytes = buffer.getInt(); + buffer.position(buffer.position() + sortDataLengthInBytes); + } + if (groupDataFeature) { int len = buffer.getInt(); groupData = new byte[len]; @@ -142,36 +151,28 @@ public class QueryResultPacket extends Packet { public static final int QRF_GROUPDATA = 0x00000200; public static final int QRF_PROPERTIES = 0x00000400; - /** - * Sets the features of this package. - * Features are either encoded by different package codes - * or by a feature int, for reasons not easily comprehended. - */ + /** Decodes the feature int of this package data into boolean feature fields */ private void decodeFeatures(IntBuffer buffer) { - switch (getCode()) { - case 217: - int features=buffer.get(); - mldFeature = (QRF_MLD & features) != 0; - // Data given by sortFeature not currently used by QRS: - // sortFeature = (QRF_SORTDATA & features) != 0; - coverageNodes = (QRF_COVERAGE_NODES & features) != 0; - groupDataFeature = (QRF_GROUPDATA & features) != 0; - propsFeature = (QRF_PROPERTIES & features) != 0; - break; - default: - throw new RuntimeException("Programming error, packet " + getCode() + "Not expected."); - } + int features=buffer.get(); + mldFeature = (QRF_MLD & features) != 0; + sortData = (QRF_SORTDATA & features) != 0; + coverageNodes = (QRF_COVERAGE_NODES & features) != 0; + groupDataFeature = (QRF_GROUPDATA & features) != 0; + propsFeature = (QRF_PROPERTIES & features) != 0; } private void decodeDocuments(ByteBuffer buffer, int documentCount) { - for (int i=0; i