diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/prelude/fastsearch')
36 files changed, 4165 insertions, 0 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java new file mode 100644 index 00000000000..44107499b40 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a byte field in the result set + * + */ + +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.search.result.NanNumber; +import com.yahoo.data.access.Inspector; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ +public class ByteField extends DocsumField { + static final byte EMPTY_VALUE = Byte.MIN_VALUE; + + public ByteField(String name) { + super(name); + } + + private Object convert(byte value) { + if (value == EMPTY_VALUE) { + return NanNumber.NaN; + } else { + return Byte.valueOf(value); + } + } + + public Object decode(ByteBuffer b) { + return convert(b.get()); + } + + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + public int getLength(ByteBuffer b) { + int offset = b.position(); + final int bytelength = Byte.SIZE >> 3; + b.position(offset + bytelength); + return bytelength; + } + + public Object convert(Inspector value) { + return convert((byte)value.asLong(EMPTY_VALUE)); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java new file mode 100644 index 00000000000..fdc76835e1e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import com.yahoo.fs4.Packet; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.fs4.QueryResultPacket; +import com.yahoo.search.Query; +import com.yahoo.processing.request.CompoundName; + + +/** + * The cache control logic for FastSearcher + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class CacheControl { + + private static final CompoundName nocachewrite=new CompoundName("nocachewrite"); + + /** Whether this CacheControl actually should cache hits at all. */ + private final boolean activeCache; + + /** Direct unsychronized cache access */ + private final PacketCache packetCache; + + public CacheControl(int sizeMegaBytes, double cacheTimeOutSeconds) { + activeCache = sizeMegaBytes > 0 && cacheTimeOutSeconds > 0.0d; + if (activeCache) { + packetCache = new PacketCache(sizeMegaBytes, 0, cacheTimeOutSeconds); + } else { + packetCache = null; + } + } + + /** Returns the capacity of the packet cache in megabytes */ + public final int capacity() { + return packetCache.getCapacity(); + } + + public final boolean useCache(Query query) { + return (activeCache && !query.getNoCache()); + } + + public final PacketWrapper lookup(CacheKey key, Query query) { + if ((key != null) && useCache(query)) { + long now = System.currentTimeMillis(); + synchronized (packetCache) { + return packetCache.get(key, now); + } + } + return null; + } + + // updates first phase in multi phase search + void updateCacheEntry(CacheKey key, Query query, QueryResultPacket resultPacket) { + long oldTimestamp; + if (!activeCache) return; + + PacketWrapper wrapper = lookup(key, query); + if (wrapper == null) return; + + // The timestamp is owned by the QueryResultPacket, this is why this + // update method puts entries into the cache differently from elsewhere + oldTimestamp = wrapper.getTimestamp(); + wrapper = (PacketWrapper) wrapper.clone(); + wrapper.addResultPacket(resultPacket); + synchronized (packetCache) { + packetCache.put(key, wrapper, oldTimestamp); + } + } + + // updates phases after first phase phase in multi phase search + void updateCacheEntry(CacheKey key, Query query, DocsumPacketKey[] packetKeys, Packet[] packets) { + if (!activeCache) return; + + PacketWrapper wrapper = lookup(key, query); + if (wrapper== null) return; + + wrapper = (PacketWrapper) wrapper.clone(); + wrapper.addDocsums(packetKeys, packets); + synchronized (packetCache) { + packetCache.put(key, wrapper, wrapper.getTimestamp()); + } + } + + void cache(CacheKey key, Query query, DocsumPacketKey[] packetKeys, Packet[] packets) { + if ( ! activeCache) return; + + if (query.getNoCache()) return; + if (query.properties().getBoolean(nocachewrite)) return; + + PacketWrapper wrapper = lookup(key, query); + if (wrapper == null) { + wrapper = new PacketWrapper(key, packetKeys,packets); + long now = System.currentTimeMillis(); + synchronized (packetCache) { + packetCache.put(key, wrapper, now); + } + } else { + wrapper = (PacketWrapper) wrapper.clone(); + wrapper.addResultPacket((QueryResultPacket) packets[0]); + wrapper.addDocsums(packetKeys, packets, 1); + synchronized (packetCache) { + packetCache.put(key, wrapper, wrapper.getTimestamp()); + } + } + } + + /** Test method. */ + public void clear() { + if (packetCache != null) { + packetCache.clear(); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheKey.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheKey.java new file mode 100644 index 00000000000..cd330603b3d --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheKey.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import java.util.Arrays; + +import com.yahoo.collections.BobHash; +import com.yahoo.fs4.QueryPacket; + + +/** + * The key used in the packet cache. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class CacheKey { + private int hashCode; + private byte[] serialized = null; + + /** + * Create a cache key from the query packet. + */ + public CacheKey(QueryPacket queryPacket) { + if (!queryPacket.isEncoded()) { + queryPacket.allocateAndEncode(0); + } + this.serialized = queryPacket.getOpaqueCacheKey(); + hashCode = calculateHashCode(); + } + + private int calculateHashCode() { + return BobHash.hash(serialized, 0); + } + + public boolean equals(Object o) { + if (o == null) { + return false; + } + if (!(o instanceof CacheKey)) { + return false; + } + + CacheKey k = (CacheKey) o; + return Arrays.equals(serialized, k.serialized); + // // The following is used for detailed debugging + // boolean state = true; + // if (serialized.length != k.serialized.length) { + // System.out.println("this " + serialized.length + " other " + k.serialized.length); + // return false; + // } + // System.out.println("start of arrays"); + // for (int i = 0; i < serialized.length; ++i) { + // System.out.print("serialized " + serialized[i] + " " + k.serialized[i]); + // if (serialized[i] != k.serialized[i]) { + // System.out.println(" diff at index " + i); + // state = false; // want to see all the data + // } else { + // System.out.println(""); + // } + // } + // return state; + } + + public int hashCode() { + return hashCode; + } + + public byte[] getCopyOfFullKey() { + return Arrays.copyOf(serialized, serialized.length); + } + + /** + * Return an estimate of the memory used by this object. Ie the sum of + * the internal data fields. + */ + public int byteSize() { + // 4 = sizeOf(hashCode) + return serialized.length + 4; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheParams.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheParams.java new file mode 100644 index 00000000000..f7714ce1457 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheParams.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +/** + * Helper class for carrying around cache-related + * config parameters to the FastSearcher class. + * + * @author arnej27959 + */ +public class CacheParams { + public int cacheMegaBytes = 0; + public double cacheTimeOutSeconds = 0; + public CacheControl cacheControl = null; + + public CacheParams(int megabytes, double timeoutseconds) { + this.cacheMegaBytes = megabytes; + this.cacheTimeOutSeconds = timeoutseconds; + } + + public CacheParams(CacheControl cacheControl) { + this.cacheControl = cacheControl; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java new file mode 100644 index 00000000000..d5a17060dd6 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.container.search.LegacyEmulationConfig; + +/** + * Helper class for carrying around cluster-related + * config parameters to the FastSearcher class. + * + * @author arnej27959 + */ +public class ClusterParams { + public final int clusterNumber; + public final String searcherName; + public final int rowBits; + public final LegacyEmulationConfig emulation; + + /** + * for compatibility + **/ + public ClusterParams(int number, String name, int rowbits) { + this(number, name, rowbits, new LegacyEmulationConfig()); + } + + /** + * for testcases only + **/ + public ClusterParams(String name) { + this(0, name, 0); + } + + /** + * make up full ClusterParams + **/ + public ClusterParams(int number, String name, int rowbits, LegacyEmulationConfig cfg) { + this.clusterNumber = number; + this.searcherName = name; + this.rowBits = rowbits; + this.emulation = cfg; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java new file mode 100644 index 00000000000..0e54adae932 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a data field in the result set. a data field + * is basically the same thing as a string field, only that we + * treat it like a raw buffer. Well we SHOULD. we don't actually + * do so. yet. we should probably do some defensive copying and + * return a ByteBuffer...hmm... + * + */ + +package com.yahoo.prelude.fastsearch; + +import java.nio.ByteBuffer; + +import com.yahoo.prelude.hitfield.RawData; +import com.yahoo.data.access.simple.Value; +import com.yahoo.data.access.Inspector; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ +public class DataField extends DocsumField implements VariableLengthField { + public DataField(String name) { + super(name); + } + + private Object convert(byte[] value) { + return new RawData(value); + } + + @Override + public Object decode(ByteBuffer b) { + int len = ((int) b.getShort()) & 0xffff; + + byte[] tmp = new byte[len]; + b.get(tmp); + return convert(tmp); + } + + @Override + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + @Override + public String toString() { + return "field " + getName() + " type data"; + } + + @Override + public int getLength(ByteBuffer b) { + int offset = b.position(); + int len = ((int) b.getShort()) & 0xffff; + b.position(offset + len + (Short.SIZE >> 3)); + return len + (Short.SIZE >> 3); + } + + @Override + public int sizeOfLength() { + return Short.SIZE >> 3; + } + + @Override + public Object convert(Inspector value) { + return convert(value.asData(Value.empty().asData())); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/Docsum.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Docsum.java new file mode 100644 index 00000000000..2941baf40f5 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Docsum.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + + +/** + * An instance of a document summary, backed by binary data, which decodes and returns fields on request, + * using the (shared) definition of this docsum. + * + * @author <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Docsum { + + private final DocsumDefinition definition; + private final byte[] packet; + /** The offsets into the packet data of each field, given the fields index, computed lazily */ + private final int[] fieldOffsets; + /** The largest stored offset */ + private int largestStoredOffset=-1; + + public Docsum(DocsumDefinition definition, byte[] packet) { + this.definition = definition; + this.packet = packet; + fieldOffsets=new int[definition.getFieldCount()]; + } + + public DocsumDefinition getDefinition() { return definition; } + + public Integer getFieldIndex(String fieldName) { + return definition.getFieldIndex(fieldName); + } + + public Object decode(int fieldIndex) { + ByteBuffer b=packetAsBuffer(); + setAndReturnOffsetToField(b, fieldIndex); + return definition.getField(fieldIndex).decode(b); + } + + /** Fetches the field as raw utf-8 if it is a text field. Returns null otherwise */ + public FastHit.RawField fetchFieldAsUtf8(int fieldIndex) { + DocsumField dataType = definition.getField(fieldIndex); + if ( ! (dataType instanceof LongstringField || dataType instanceof XMLField || dataType instanceof StringField)) + return null; + + ByteBuffer b=packetAsBuffer(); + DocsumField field = definition.getField(fieldIndex); + int fieldStart = setAndReturnOffsetToField(b, fieldIndex); // set buffer.pos = start of field + if (field.isCompressed(b)) return null; + int length = field.getLength(b); // scan to end of field + if (field instanceof VariableLengthField) { + int fieldLength = ((VariableLengthField) field).sizeOfLength(); + b.position(fieldStart + fieldLength); // reset to start of field + length -= fieldLength; + } else { + b.position(fieldStart); // reset to start of field + } + byte[] bufferView = new byte[length]; + b.get(bufferView); + return new FastHit.RawField(dataType, bufferView); + } + + public ByteBuffer packetAsBuffer() { + ByteBuffer buffer = ByteBuffer.wrap(packet); + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.getInt(); // Skip class id + return buffer; + } + + /** Returns the offset of a given field in the buffer, and sets the position of the buffer to that field start */ + private int setAndReturnOffsetToField(ByteBuffer b, int fieldIndex) { + // find and store missing offsets up to fieldIndex + if (largestStoredOffset<0) { // initial case + fieldOffsets[0]=b.position(); + largestStoredOffset++; + } + while (largestStoredOffset < fieldIndex) { // induction + int offsetOfLargest=fieldOffsets[largestStoredOffset]; + b.position(offsetOfLargest); + fieldOffsets[largestStoredOffset+1]=offsetOfLargest+definition.getField(largestStoredOffset).getLength(b); + largestStoredOffset++; + } + + // return the stored offset + int offset=fieldOffsets[fieldIndex]; + b.position(offset); + return offset; + } + + public String toString() { + return "docsum [definition: " + definition + "]"; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java new file mode 100644 index 00000000000..bef0069d525 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.container.search.LegacyEmulationConfig; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A docsum definition which knows how to decode a certain class of document + * summaries. The docsum definition has a name and a list of field definitions + * + * @author bratseth + * @author Bjørn Borud + */ +public class DocsumDefinition { + + private String name; + private final List<DocsumField> fields; + + /** True if this contains dynamic fields */ + private boolean dynamic = false; + + // Mapping between field names and their index in this.fields + private final Map<String,Integer> fieldNameToIndex; + + DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config, LegacyEmulationConfig emulConfig) { + this.name = config.name(); + List<DocsumField> fieldsBuilder = new ArrayList<>(); + Map<String,Integer> fieldNameToIndexBuilder = new HashMap<>(); + + for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { + // no, don't switch the order of the two next lines :) + fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); + fieldsBuilder.add(DocsumField.create(field.name(), field.type(), emulConfig)); + if (field.dynamic()) + dynamic = true; + } + fields = ImmutableList.copyOf(fieldsBuilder); + fieldNameToIndex = ImmutableMap.copyOf(fieldNameToIndexBuilder); + } + + /** Returns the field at this index, or null if none */ + public DocsumField getField(int fieldIndex) { + if (fieldIndex >= fields.size()) return null; + return fields.get(fieldIndex); + } + + /** Returns the index of a field name */ + public Integer getFieldIndex(String fieldName) { + return fieldNameToIndex.get(fieldName); + } + + @Override + public String toString() { + return "docsum definition '" + getName() + "'"; + } + + public String getName() { + return name; + } + + public int getFieldCount() { + return fields.size(); + } + + public List<DocsumField> getFields() { + return fields; + } + + /** Returns whether this summary contains one or more dynamic fields */ + public boolean isDynamic() { + return dynamic; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java new file mode 100644 index 00000000000..2f0768d4e8b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Slime; +import com.yahoo.data.access.slime.SlimeAdapter; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.prelude.ConfigurationException; +import com.yahoo.container.search.LegacyEmulationConfig; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; + +/** + * A set of docsum definitions + * + * @author bratseth + * @author Bjørn Borud + */ +public final class DocsumDefinitionSet { + public static final int SLIME_MAGIC_ID = 0x55555555; + private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); + + private final HashMap<Long, DocsumDefinition> definitions = new HashMap<>(); + private final HashMap<String, DocsumDefinition> definitionsByName = new HashMap<>(); + private final LegacyEmulationConfig emulationConfig; + + public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { + this.emulationConfig = new LegacyEmulationConfig(); + configure(config); + } + + public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + this.emulationConfig = emulConfig; + configure(config); + } + + /** Returns a docsum definition by id + * @param id document summary class id + * @return a DocsumDefinition for the id, if found. + */ + public final DocsumDefinition getDocsumDefinition(long id) { + return definitions.get(new Long(id)); + } + + /** + * Returns a docsum definition by name, or null if not found + * + * @param name the name of the summary class to use, or null to use the name "default" + * @return the summary class found, or null if none + */ + public final DocsumDefinition getDocsumDefinition(String name) { + if (name == null) + name="default"; + return definitionsByName.get(name); + } + + /** + * Makes data available for decoding for the given hit. + * + * @param summaryClass the requested summary class + * @param data docsum data from backend + * @param hit the Hit corresponding to this document summary + * @throws ConfigurationException if the summary class of this hit is missing + */ + public final void lazyDecode(String summaryClass, byte[] data, FastHit hit) { + ByteBuffer buffer = ByteBuffer.wrap(data); + buffer.order(ByteOrder.LITTLE_ENDIAN); + long docsumClassId = buffer.getInt(); + if (docsumClassId != SLIME_MAGIC_ID) { + DocsumDefinition docsumDefinition = lookupDocsum(docsumClassId); + Docsum docsum = new Docsum(docsumDefinition, data); + hit.addSummary(docsum); + } else { + DocsumDefinition docsumDefinition = lookupDocsum(summaryClass); + Slime value = BinaryFormat.decode(buffer.array(), buffer.arrayOffset()+buffer.position(), buffer.remaining()); + hit.addSummary(docsumDefinition, new SlimeAdapter(value.get())); + } + } + + private DocsumDefinition lookupDocsum(long docsumClassId) { + DocsumDefinition docsumDefinition = getDocsumDefinition(docsumClassId); + if (docsumDefinition == null) { + throw new ConfigurationException("Received hit with summary id " + docsumClassId + + ", but this summary class is not in current summary config (" + toString() + ")" + + " (that is, the system is in an inconsistent state)"); + } + return docsumDefinition; + } + + private DocsumDefinition lookupDocsum(String summaryClass) { + DocsumDefinition ds = definitionsByName.get(summaryClass); + if (ds == null) { + ds = definitionsByName.get("default"); + } + if (ds == null) { + throw new ConfigurationException("Fetched hit with summary class " + summaryClass + + ", but this summary class is not in current summary config (" + toString() + ")" + + " (that is, you asked for something unknown, and no default was found)"); + } + return ds; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + Set<Map.Entry<Long, DocsumDefinition>> entrySet = definitions.entrySet(); + boolean first = true; + for (Iterator<Map.Entry<Long, DocsumDefinition>> itr = entrySet.iterator(); itr.hasNext(); ) { + if (!first) { + sb.append(","); + } else { + first = false; + } + Map.Entry<Long, DocsumDefinition> entry = itr.next(); + sb.append("[").append(entry.getKey()).append(",").append(entry.getValue().getName()).append("]"); + } + return sb.toString(); + } + + public int size() { + return definitions.size(); + } + + private void configure(DocumentdbInfoConfig.Documentdb config) { + for (int i = 0; i < config.summaryclass().size(); ++i) { + DocumentdbInfoConfig.Documentdb.Summaryclass sc = config.summaryclass(i); + DocsumDefinition docSumDef = new DocsumDefinition(sc, emulationConfig); + definitions.put((long) sc.id(), docSumDef); + definitionsByName.put(sc.name(), docSumDef); + } + if (definitions.size() == 0) { + log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); + } + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java new file mode 100644 index 00000000000..3aa02f57a1e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; +import com.yahoo.data.access.Inspector; +import com.yahoo.container.search.LegacyEmulationConfig; + +import com.yahoo.log.LogLevel; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public abstract class DocsumField { + + private static final Logger log = Logger.getLogger(DocsumField.class.getName()); + private static FieldFactory fieldFactory; + + private static class FieldFactory { + Map<String, Constructor<? extends DocsumField>> constructors = new HashMap<>(); + + void put(final String typename, + final Class<? extends DocsumField> fieldClass) + throws NoSuchMethodException, SecurityException + { + final Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class); + constructors.put(typename, constructor); + } + + DocsumField create(final String typename, final String name, final LegacyEmulationConfig emulConfig) + throws InstantiationException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException + { + DocsumField f = constructors.get(typename).newInstance(name); + f.emulConfig = emulConfig; + return f; + } + } + private LegacyEmulationConfig emulConfig; + final LegacyEmulationConfig getEmulConfig() { return emulConfig; } + + static { + fieldFactory = new FieldFactory(); + + try { + fieldFactory.put("byte", ByteField.class); + fieldFactory.put("short", ShortField.class); + fieldFactory.put("integer", IntegerField.class); + fieldFactory.put("int64", Int64Field.class); + fieldFactory.put("float", FloatField.class); + fieldFactory.put("double", DoubleField.class); + fieldFactory.put("string", StringField.class); + fieldFactory.put("data", DataField.class); + fieldFactory.put("longstring", LongstringField.class); + fieldFactory.put("longdata", LongdataField.class); + fieldFactory.put("jsonstring", StructDataField.class); + fieldFactory.put("featuredata", FeatureDataField.class); + fieldFactory.put("xmlstring", XMLField.class); + } catch (final Exception e) { + log.log(LogLevel.ERROR, + "Could not initialize docsum decoding properly.", e); + } + } + + protected String name; + + protected DocsumField(final String name) { + this.name = name; + } + + /* for unit test only */ + static DocsumField create(final String name, final String typename) { + return create(name, typename, new LegacyEmulationConfig()); + } + + public static DocsumField create(final String name, final String typename, LegacyEmulationConfig emulConfig) { + try { + return fieldFactory.create(typename, name, emulConfig); + } catch (final Exception e) { + throw new RuntimeException("Unknown field type '" + typename + "'", e); + } + } + + public String getName() { + return name; + } + + public boolean isCompressed(final ByteBuffer b) { + return false; + } + + /** + * Decode the field at the current buffer position into the fast hit. + */ + public abstract Object decode(ByteBuffer b, FastHit hit); + + /** + * Decode the field at the current buffer position and simply return the + * value. + */ + public abstract Object decode(ByteBuffer b); + + /** + * Get the number of bytes this field occupies in the given buffer and set + * the position of the first byte after this field. + */ + public abstract int getLength(ByteBuffer b); + + /** + * Convert a generic value into an object of the appropriate type + * for this field. + **/ + public abstract Object convert(Inspector value); +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumPacketKey.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumPacketKey.java new file mode 100644 index 00000000000..1e76207e370 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumPacketKey.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.document.GlobalId; + + +/** + * Key for each entry in the packetcache. + * + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00f8lster Lidal</a> + */ +public class DocsumPacketKey { + private GlobalId globalId; + private int partid; + private int docstamp; + private String summaryClass; + + private static boolean strEquals(String a, String b) { + if (a == null || b == null) { + return (a == null && b == null); + } + return a.equals(b); + } + + private static int strHashCode(String s) { + if (s == null) { + return 0; + } + return s.hashCode(); + } + + public DocsumPacketKey(GlobalId globalId, int partid, String summaryClass) { + this.globalId = globalId; + this.partid = partid; + this.summaryClass = summaryClass; + } + + public GlobalId getGlobalId() { + return globalId; + } + + public int getPartid() { + return partid; + } + + public boolean equals(Object o) { + if (o instanceof DocsumPacketKey) { + DocsumPacketKey other = (DocsumPacketKey) o; + + if (globalId.equals(other.getGlobalId()) + && partid == other.getPartid() + && strEquals(summaryClass, other.summaryClass)) + { + return true; + } + } + return false; + } + + public int hashCode() { + return globalId.hashCode() + 10 * partid + strHashCode(summaryClass); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java new file mode 100644 index 00000000000..c48a8804f9f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.container.search.LegacyEmulationConfig; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Representation of a back-end document database. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DocumentDatabase { + + // TODO: What about name conflicts when different search defs have the same rank profile/docsum? + + public static final String MATCH_PROPERTY = "match"; + public static final String SEARCH_DOC_TYPE_KEY = "documentdb.searchdoctype"; + + private final String name; + private final DocsumDefinitionSet docsumDefSet; + + private final Map<String, RankProfile> rankProfiles; + + public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb, LegacyEmulationConfig emulConfig) { + this.name = documentDb.name(); + this.docsumDefSet = new DocsumDefinitionSet(documentDb, emulConfig); + this.rankProfiles = ImmutableMap.copyOf(toRankProfiles(documentDb.rankprofile())); + } + + public String getName() { + return name; + } + + public DocsumDefinitionSet getDocsumDefinitionSet() { + return docsumDefSet; + } + + /** Returns an unmodifiable map of all the rank profiles in this indexed by rank profile name */ + public Map<String, RankProfile> rankProfiles() { return rankProfiles; } + + private Map<String, RankProfile> toRankProfiles(List<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { + Map<String, RankProfile> rankProfiles = new HashMap<>(); + for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) { + rankProfiles.put(c.name(), new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); + } + return rankProfiles; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DoubleField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DoubleField.java new file mode 100644 index 00000000000..d42d5567718 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DoubleField.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.search.result.NanNumber; +import com.yahoo.data.access.Inspector; + +/** + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00f8lster Lidal</a> + */ +public class DoubleField extends DocsumField { + static final double EMPTY_VALUE = Double.NaN; + + public DoubleField(String name) { + super(name); + } + + private Object convert(double value) { + if (Double.isNaN(value)) { + return NanNumber.NaN; + } else { + return Double.valueOf(value); + } + } + + public Object decode(ByteBuffer b) { + return convert(b.getDouble()); + } + + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + public int getLength(ByteBuffer b) { + int offset = b.position(); + final int byteLength = Double.SIZE >> 3; + b.position(offset + byteLength); + return byteLength; + } + + public Object convert(Inspector value) { + return convert(value.asDouble(EMPTY_VALUE)); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java new file mode 100644 index 00000000000..1aa226dbeb8 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.container.Server; +import com.yahoo.container.search.Fs4Config; +import com.yahoo.fs4.mplex.Backend; +import com.yahoo.fs4.mplex.ConnectionPool; +import com.yahoo.fs4.mplex.ListenerPool; +import com.yahoo.io.Connection; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Provider for {@link com.yahoo.fs4.mplex.ListenerPool}. All users will get the same pool instance. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.4.0 + */ +public class FS4ResourcePool extends AbstractComponent { + private static final Logger logger = Logger.getLogger(FS4ResourcePool.class.getName()); + private static final AtomicInteger instanceCounter = new AtomicInteger(0); + private final int instanceId; + private final ListenerPool listeners; + private final Timer timer = new Timer(); // This is a timer for cleaning the closed connections + private Map<String, Backend> connectionPoolMap = new HashMap<>(); + private final ExecutorService executor; + private final ScheduledExecutorService scheduledExecutor; + + public FS4ResourcePool(Fs4Config fs4Config) { + instanceId = instanceCounter.getAndIncrement(); + logger.log(Level.INFO, "Constructing an FS4ResourcePool with id '" + instanceId + "' with config '" + fs4Config.toString() + "'"); + String name = "FS4-" + instanceId; + listeners = new ListenerPool(name, fs4Config.numlistenerthreads()); + executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(name)); + scheduledExecutor = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory(name + ".scheduled")); + } + + public ExecutorService getExecutor() { + return executor; + } + public ScheduledExecutorService getScheduledExecutor() { + return scheduledExecutor; + } + public Backend getBackend(String host, int port) { + + String key = host + ":" + port; + synchronized (connectionPoolMap) { + Backend pool = connectionPoolMap.get(key); + if (pool == null) { + pool = new Backend(host, port, Server.get().getServerDiscriminator(), listeners, new ConnectionPool(timer)); + connectionPoolMap.put(key, pool); + } + return pool; + } + } + + @Override + public void deconstruct() { + logger.log(Level.INFO, "Deconstructing FS4ResourcePool with id '" + instanceId + "'."); + super.deconstruct(); + listeners.close(); + timer.cancel(); + for (Backend backend : connectionPoolMap.values()) { + backend.shutdown(); + backend.close(); + } + executor.shutdown(); + scheduledExecutor.shutdown(); + try { + executor.awaitTermination(10, TimeUnit.SECONDS); + scheduledExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + logger.warning("Executors failed terminating within timeout of 10 seconds : " + e); + } + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java new file mode 100644 index 00000000000..ee3f9ac0583 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -0,0 +1,442 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.google.common.annotations.Beta; +import com.yahoo.document.GlobalId; +import com.yahoo.fs4.QueryPacketData; +import com.yahoo.net.URI; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; +import com.yahoo.data.access.simple.Value.StringValue; + +/** + * A regular hit from a Vespa backend + * + * @author bratseth + * @author steinar + */ +public class FastHit extends Hit { + + public static final String SUMMARY = "summary"; + + private static final long serialVersionUID = 298098891191029589L; + + /** The global id of this document in the backend node which produced it */ + private GlobalId globalId = new GlobalId(new byte[GlobalId.LENGTH]); + + /** Part ID */ + private int partId; + + /** DistributionKey (needed to generate getDocsumPacket, for two-phase search) */ + private int distributionKey = 0; + + /** The index uri of this. Lazily set */ + private URI indexUri = null; + + /** + * The number of least significant bits in the part id which specifies the + * row in the search cluster which produced this hit. The other bits + * specifies the column. 0 if not known. + */ + private int rowBits = 0; + + /** + * Whether or not to ignore the row bits. If this is set, FastSearcher is + * allowed to choose an appropriate row. + */ + private boolean ignoreRowBits = false; + + /** + * Whether to use the row number in the index uri, see FastSearcher for + * details + */ + private boolean useRowInIndexUri = true; + + private transient QueryPacketData queryPacketData = null; + private transient CacheKey cacheKey = null; + + /** + * Creates an empty and temporarily invalid summary hit + */ + public FastHit() { } + + public FastHit(String uri, double relevancy) { + this(uri, relevancy, null); + } + + public FastHit(String uri, double relevance, String source) { + setId(uri); + super.setField("uri", uri); + setRelevance(new Relevance(relevance)); + setSource(source); + types().add(SUMMARY); + setPartId(0, 0); + } + + public String toString() { + return super.toString() + " [fasthit, globalid: " + globalId + ", partId: " + + partId + ", distributionkey: " + distributionKey + "]"; + } + + public static String asHexString(GlobalId gid) { + StringBuilder sb = new StringBuilder(); + byte[] rawGid = gid.getRawId(); + for (byte b : rawGid) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + sb.append('0'); + } + sb.append(hex); + } + return sb.toString(); + } + + @Override + public int hashCode() { + if (getId() == null) { + throw new IllegalStateException("This hit must have a 'uri' field, and this fild must be filled through " + + "Execution.fill(Result)) before hashCode() is accessed."); + } else { + return super.hashCode(); + } + } + + @Override + public URI getId() { + return getUri(); // Make sure we decode it if the id is encoded + } + + /** + * Returns the explicitly set uri if available, returns + * "index:[source]/[partid]/[id]" otherwise + * @return uri of hit + */ + public URI getUri() { + URI uri = super.getId(); + if (uri != null) return uri; + + // TODO: Remove, this should be one of the last vestiges of URL field magic + if (fields().containsKey("uri")) { + // trigger decoding + Object o = getField("uri"); + setId(o.toString()); + return super.getId(); + } + + return getIndexUri(); + } + + /** + * The uri of the index location of this hit + * ("index:[source]/[partid]/[id]"). This is the uri if no other uri is + * assigned + * @return uri to the index. + */ + public URI getIndexUri() { + if (indexUri != null) return indexUri; + + String rowString = "-"; + if (useRowInIndexUri) + rowString = String.valueOf(getRow()); + + return new URI("index:" + getSourceNumber() + "/" + getColumn() + "/" + rowString + "/" + asHexString(getGlobalId())); + } + + /** Returns the global id of this document in the backend node which produced it */ + public GlobalId getGlobalId() { + return globalId; + } + + public void setGlobalId(GlobalId globalId) { + this.globalId = globalId; + } + + public int getPartId() { + return partId; + } + + /** + * Sets the part id number, which specifies the node where this hit is + * found. The row count is used to decode the part id into a column and a + * row number: the number of n least significant bits required to hold the + * highest row number are the row bits, the rest are column bits. + * + * @param partId partition id + * @param rowBits number of bits to encode row number + */ + public void setPartId(int partId, int rowBits) { + this.partId = partId; + this.rowBits = rowBits; + } + + /** + * + * @param useRowInIndexUri Sets whether to use the row in the index uri. See FastSearcher for details. + */ + public void setUseRowInIndexUri(boolean useRowInIndexUri) { + this.useRowInIndexUri = useRowInIndexUri; + } + + /** + * @return Returns the column number where this hit originated, or partId if not known + */ + public int getColumn() { + return partId >>> rowBits; + } + + /** + * @return the row number where this hit originated, or 0 if not known + * */ + public int getRow() { + if (rowBits == 0) { + return 0; + } + + return partId & ((1 << rowBits) - 1); + } + + /** + * <p>Returns a field value from this Hit. The value is either a stored value from the Document represented by + * this Hit, or a generated value added during later processing.</p> + * + * <p>The values available from the matching Document are a <i>subset</i> of the values set in the document, + * determined by the {@link #getFilled() filled} status of this Hit. More fields may be requested by requesting + * further filling.</p> + * + * <p>Lookups on names which does not exists in the document and is not added by later processing + * return null.</p> + * + * <p>Lookups on fields which exist in the document, in a summary class which is already requested + * filled returns the following types, even when the field has no actual value:</p> + * + * <ul> + * <li><b>Dynamic summary string fields</b>: A Java String before JuniperSearcher and a HitField after.</li> + * <li><b>string/uri/content</b>: A Java String.<br> + * The empty string ("") if no value is assigned in the document. + * + * <li><b>Numerics</b>: The corresponding numeric Java type.<br> + * If the field has <i>no value</i> assigned in the document, + * the special numeric {@link com.yahoo.search.result.NanNumber#NaN} is returned. + * + * <li><b>raw</b>: A {@link com.yahoo.prelude.hitfield.RawData} instance + * + * <li><b>multivalue fields</b>: A {@link com.yahoo.prelude.hitfield.JSONString} instance + * </ul> + */ + @Override + public Object getField(String key) { + Object value = super.getField(key); + + if (value instanceof LazyValue) { + return getAndCacheLazyValue(key, (LazyValue) value); + } else { + return value; + } + } + + private Object getAndCacheLazyValue(String key, LazyValue value) { + Object forcedValue = value.getValue(key); + setField(key, forcedValue); + return forcedValue; + } + + /** Returns false - this is a concrete hit containing requested content */ + public boolean isMeta() { + return false; + } + + /** + * Only needed when fetching summaries in 2 phase. + * + * @return distribution key of node where the hit originated from + */ + public int getDistributionKey() { + return distributionKey; + } + + /** + * Only needed when fetching summaries in 2 phase. + * @param distributionKey Of node where you find this hit. + */ + public void setDistributionKey(int distributionKey) { + this.distributionKey = distributionKey; + } + + public void addSummary(Docsum docsum) { + LazyDocsumValue lazyDocsumValue = new LazyDocsumValue(docsum); + for (DocsumField field : docsum.getDefinition().getFields()) { + setDocsumFieldIfNotPresent(field.getName(), lazyDocsumValue); + } + } + + void addSummary(DocsumDefinition docsumDef, Inspector value) { + for (DocsumField field : docsumDef.getFields()) { + String fieldName = field.getName(); + if (value.type() == Type.STRING && + (field instanceof LongstringField || + field instanceof StringField || + field instanceof XMLField)) + { + setDocsumFieldIfNotPresent(fieldName, new LazyString(field, value)); + } else { + Inspector f = value.field(fieldName); + if (field.getEmulConfig().forceFillEmptyFields() || f.valid()) { + setDocsumFieldIfNotPresent(fieldName, field.convert(f)); + } + } + } + } + + private void setDocsumFieldIfNotPresent(String fieldName, Object value) { + if (super.getField(fieldName) == null) { + setField(fieldName, value); + } + } + + /** + * Set a field to behave like a string type summary field, not decoding raw + * data till actually used. Added to make testing lazy docsum functionality + * easier. This is not a method to be used for efficiency, as it causes + * object allocations. + * + * @param fieldName + * the name of the field to insert undecoded UTF-8 into + * @param value + * an array of valid UTF-8 data + */ + @Beta + public void setLazyStringField(String fieldName, byte[] value) { + setField(fieldName, new LazyString(new StringField(fieldName), new StringValue(value))); + } + + public static final class RawField { + private final boolean needXmlEscape; + + private final byte[] contents; + + public RawField(DocsumField fieldType, byte[] contents) { + needXmlEscape = ! (fieldType instanceof XMLField); + this.contents = contents; + } + public RawField(byte [] contents) { + needXmlEscape = true; + this.contents = contents; + } + + public byte [] getUtf8() { return contents; } + public boolean needXmlEscape() { return needXmlEscape; } + } + + /** + * Add the binary data common for the query packet to a Vespa backend and a + * summary fetch packet to a Vespa backend. This method can only be called + * once for a single hit. + * + * @param queryPacketData binary data from a query packet resulting in this hit + * @throws IllegalStateException if the method is called more than once + * @throws NullPointerException if trying to set query packet data to null + */ + public void setQueryPacketData(QueryPacketData queryPacketData) { + if (this.queryPacketData != null) + throw new IllegalStateException("Query packet data already set to " + + this.queryPacketData + ", tried to set it to " + queryPacketData); + if (queryPacketData == null) + throw new NullPointerException("Query packet data reference can not be set to null."); + this.queryPacketData = queryPacketData; + } + + /** + * Fetch binary data from the query packet which produced this hit. These + * data may not be available, this method will then return null. + * + * @return wrapped binary data from a query packet, or null + */ + public QueryPacketData getQueryPacketData() { + return queryPacketData; + } + + public void clearQueryPacketData() { + queryPacketData = null; + } + + CacheKey getCacheKey() { + return cacheKey; + } + + void setCacheKey(CacheKey cacheKey) { + this.cacheKey = cacheKey; + } + + public void setIgnoreRowBits(boolean ignoreRowBits) { + this.ignoreRowBits = ignoreRowBits; + } + + public boolean shouldIgnoreRowBits() { + return ignoreRowBits; + } + + public boolean fieldIsNotDecoded(String name) { + return super.getField(name) instanceof LazyValue; + } + + public RawField fetchFieldAsUtf8(String fieldName) { + Object value = super.getField(fieldName); + if (value instanceof LazyValue) { + return ((LazyValue) value).getFieldAsUtf8(fieldName); + } else { + throw new IllegalStateException("Field " + fieldName + " has already been decoded:" + value); + } + } + + private static abstract class LazyValue { + abstract Object getValue(String fieldName); + abstract RawField getFieldAsUtf8(String fieldName); + } + + /** + * Represents a value that resides in the docsum. + */ + private static class LazyDocsumValue extends LazyValue { + private final Docsum docsum; + + LazyDocsumValue(Docsum docsum) { + this.docsum = docsum; + } + + Object getValue(String fieldName) { + return docsum.decode(getFieldIndex(fieldName)); + } + + private int getFieldIndex(String fieldName) { + Integer index = docsum.getFieldIndex(fieldName); + if (index == null) throw new AssertionError("Invalid fieldName " + fieldName); + return index; + } + + RawField getFieldAsUtf8(String fieldName) { + return docsum.fetchFieldAsUtf8(getFieldIndex(fieldName)); + } + } + + private static class LazyString extends LazyValue { + private final Inspector value; + private final DocsumField fieldType; + + LazyString(DocsumField fieldType, Inspector value) { + assert(value.type() == Type.STRING); + this.value = value; + this.fieldType = fieldType; + } + + Object getValue(String fieldName) { + return value.asString(); + } + + RawField getFieldAsUtf8(String fieldName) { + return new RawField(fieldType, value.asUtf8()); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java new file mode 100644 index 00000000000..dfca9c49cba --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -0,0 +1,566 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import java.util.Optional; + +import com.yahoo.compress.CompressionType; +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.ChannelTimeoutException; +import com.yahoo.fs4.GetDocSumsPacket; +import com.yahoo.fs4.Packet; +import com.yahoo.fs4.PingPacket; +import com.yahoo.fs4.PongPacket; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.fs4.QueryResultPacket; +import com.yahoo.fs4.mplex.Backend; +import com.yahoo.fs4.mplex.FS4Channel; +import com.yahoo.fs4.mplex.InvalidChannelException; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import edu.umd.cs.findbugs.annotations.NonNull; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Iterator; +import java.util.TimeZone; +import java.util.logging.Level; + +import static com.yahoo.container.util.Util.quote; + +/** + * The searcher which forwards queries to fdispatch nodes, using the fnet/fs4 + * network layer. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +// TODO: Clean up all the duplication in the various search methods by +// switching to doing all the error handling using exceptions below doSearch2. +// Right now half is done by exceptions handled in doSearch2 and half by setting +// errors on results and returning them. It could be handy to create a QueryHandlingErrorException +// or similar which could wrap an error message, and then just always throw that and +// catch and unwrap into a results with an error in high level methods. -Jon +public class FastSearcher extends VespaBackEndSearcher { + + /** If this is turned on this will fill summaries by dispatching directly to search nodes over RPC */ + private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); + + /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ + private final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); + + /** Used to dispatch directly to search nodes over RPC, replacing the old fnet communication path */ + private final Dispatcher dispatcher; + + /** Time (in ms) at which the index of this searcher was last modified */ + private volatile long editionTimeStamp = 0; + + /** Edition of the index */ + private int docstamp; + + private Backend backend; + + /** + * Creates a Fastsearcher. + * + * @param backend The backend object for this FastSearcher + * @param docSumParams Document summary parameters + * @param clusterParams The cluster number, and other cluster backend parameters + * @param cacheParams The size, lifetime, and controller of our cache + * @param documentdbInfoConfig Document database parameters + */ + public FastSearcher(Backend backend, Dispatcher dispatcher, SummaryParameters docSumParams, ClusterParams clusterParams, + CacheParams cacheParams, DocumentdbInfoConfig documentdbInfoConfig) { + init(docSumParams, clusterParams, cacheParams, documentdbInfoConfig); + this.backend = backend; + this.dispatcher = dispatcher; + } + + /** Clears the packet cache if the received timestamp is older than our timestamp */ + private void checkTimestamp(QueryResultPacket resultPacket) { + checkTimestamp(resultPacket.getDocstamp()); + } + + /** Clears the packet cache if the received timestamp is older than our timestamp */ + private void checkTimestamp(int newDocstamp) { + if (docstamp < newDocstamp) { + long currentTimeMillis = System.currentTimeMillis(); + + docstamp = newDocstamp; + setEditionTimeStamp(currentTimeMillis); + } + } + + private static SimpleDateFormat isoDateFormat; + + static { + isoDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + isoDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + private int countNumberOfFastHits(Result result) { + int numFastHits = 0; + + for (Iterator<com.yahoo.search.result.Hit> i = hitIterator(result); i.hasNext();) { + com.yahoo.search.result.Hit hit = i.next(); + + if (hit instanceof FastHit) { + numFastHits++; + } + } + return numFastHits; + } + + /** + * Pings the backend. Does not propagate to other searchers. + */ + @Override + public Pong ping(Ping ping, Execution execution) { + // If you want to change this code, you need to understand + // com.yahoo.prelude.cluster.ClusterSearcher.ping(Searcher) and + // com.yahoo.prelude.cluster.TrafficNodeMonitor.failed(ErrorMessage) + FS4Channel channel = backend.openPingChannel(); + + try { + PingPacket pingPacket = new PingPacket(); + pingPacket.enableActivedocsReporting(); + Pong pong = new Pong(); + + try { + boolean couldSend = channel.sendPacket(pingPacket); + if (!couldSend) { + pong.addError(ErrorMessage.createBackendCommunicationError("Could not ping in " + getName())); + return pong; + } + } catch (InvalidChannelException e) { + pong.addError(ErrorMessage.createBackendCommunicationError("Invalid channel " + getName())); + return pong; + } catch (IllegalStateException e) { + pong.addError( + ErrorMessage.createBackendCommunicationError("Illegal state in FS4: " + e.getMessage())); + return pong; + } catch (IOException e) { + pong.addError(ErrorMessage.createBackendCommunicationError("IO error while sending ping: " + e.getMessage())); + return pong; + } + // We should only get a single packet + BasicPacket[] packets; + + try { + packets = channel.receivePackets(ping.getTimeout(), 1); + } catch (ChannelTimeoutException e) { + pong.addError(ErrorMessage.createNoAnswerWhenPingingNode("timeout while waiting for fdispatch for " + getName())); + return pong; + } catch (InvalidChannelException e) { + pong.addError(ErrorMessage.createBackendCommunicationError("Invalid channel for " + getName())); + return pong; + + } + + if (packets.length == 0) { + pong.addError(ErrorMessage.createBackendCommunicationError(getName() + " got no packets back")); + return pong; + } + + if (isLoggingFine()) { + getLogger().finest("got packets " + packets.length + " packets"); + } + + try { + ensureInstanceOf(PongPacket.class, packets[0]); + } catch (TimeoutException e) { + pong.addError(ErrorMessage.createTimeout(e.getMessage())); + return pong; + } catch (IOException e) { + pong.addError(ErrorMessage.createBackendCommunicationError("Unexpected packet class returned after ping: " + e.getMessage())); + return pong; + } + pong.addPongPacket((PongPacket) packets[0]); + checkTimestamp(((PongPacket) packets[0]).getDocstamp()); + return pong; + } finally { + if (channel != null) { + channel.close(); + } + } + } + + protected void transformQuery(Query query) { + QueryRewrite.rewriteSddocname(query); + } + + @Override + public Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) { + FS4Channel channel = null; + try { + channel = backend.openChannel(); + channel.setQuery(query); + + // If not found, then fetch from the source. The call to + // insert into cache will be made from within searchTwoPhase + Result result = searchTwoPhase(channel, query, queryPacket, cacheKey); + + if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) { + // There is currently no correct choice for which + // summary class we want to fetch at this point. If we + // fetch the one selected by the user it may not + // contain the data we need. If we fetch the default + // one we end up fetching docsums twice unless the + // user also requested the default one. + fill(result, query.getPresentation().getSummary(), execution); // ARGH + } + return result; + } catch (TimeoutException e) { + return new Result(query,ErrorMessage.createTimeout(e.getMessage())); + } catch (IOException e) { + Result result = new Result(query); + if (query.getTraceLevel() >= 1) + query.trace(getName() + " error response: " + result, false, 1); + result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " failed: "+ e.getMessage())); + return result; + } finally { + if (channel != null) { + channel.close(); + } + } + } + + /** + * Only used to fill the sddocname field when using direct dispatching as that is normally done in VespaBackEndSearcher.decodeSummary + * @param result The result + */ + private void fillSDDocName(Result result) { + DocumentDatabase db = getDocumentDatabase(result.getQuery()); + for (Iterator<Hit> i = hitIterator(result); i.hasNext();) { + Hit hit = i.next(); + if (hit instanceof FastHit) { + hit.setField(Hit.SDDOCNAME_FIELD, db.getName()); + } + } + } + /** + * Perform a partial docsum fill for a temporary result + * representing a partition of the complete fill request. + * + * @param result result containing a partition of the unfilled hits + * @param summaryClass the summary class we want to fill with + **/ + protected void doPartialFill(Result result, String summaryClass) { + if (result.isFilled(summaryClass)) return; + + Query query = result.getQuery(); + traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass)); + + if (query.properties().getBoolean(dispatchSummaries)) { + CompressionType compression = + CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase()); + fillSDDocName(result); + dispatcher.fill(result, summaryClass, compression); + return; + } + + CacheKey cacheKey = null; + PacketWrapper packetWrapper = null; + if (getCacheControl().useCache(query)) { + cacheKey = fetchCacheKeyFromHits(result.hits(), summaryClass); + if (cacheKey == null) { + QueryPacket queryPacket = QueryPacket.create(query); + cacheKey = new CacheKey(queryPacket); + } + packetWrapper = cacheLookupTwoPhase(cacheKey, result,summaryClass); + } + + FS4Channel channel = backend.openChannel(); + channel.setQuery(query); + Packet[] receivedPackets; + try { + DocsumPacketKey[] packetKeys; + + if (countNumberOfFastHits(result) > 0) { + packetKeys = getPacketKeys(result, summaryClass, false); + if (packetKeys.length == 0) { + receivedPackets = new Packet[0]; + } else { + try { + receivedPackets = fetchSummaries(channel, result, summaryClass); + } catch (InvalidChannelException e) { + result.hits().addError(ErrorMessage.createBackendCommunicationError("Invalid channel " + getName() + " (summary fetch)")); + return; + } catch (ChannelTimeoutException e) { + result.hits().addError(ErrorMessage.createTimeout("timeout waiting for summaries from " + getName())); + return; + } catch (IOException e) { + result.hits().addError(ErrorMessage.createBackendCommunicationError( + "IO error while talking on channel " + getName() + " (summary fetch): " + e.getMessage())); + return; + } + if (receivedPackets.length == 0) { + result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " got no packets back (summary fetch)")); + return; + } + } + } else { + packetKeys = new DocsumPacketKey[0]; + receivedPackets = new Packet[0]; + } + + int skippedHits; + try { + skippedHits = fillHits(result, 0, receivedPackets, summaryClass); + } catch (TimeoutException e) { + result.hits().addError(ErrorMessage.createTimeout(e.getMessage())); + return; + } catch (IOException e) { + result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling hits with summary fields, source: " + getName())); + return; + } + if (skippedHits==0 && packetWrapper != null) { + cacheControl.updateCacheEntry(cacheKey, query, packetKeys, receivedPackets); + } + + if ( skippedHits>0 ) { + getLogger().info("Could not fill summary '" + summaryClass + "' for " + skippedHits + " hits for query: " + result.getQuery()); + result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit data for summary '" + summaryClass + "' for " + skippedHits + " hits")); + } + result.analyzeHits(); + + if (query.getTraceLevel() >= 3) { + int hitNumber = 0; + for (Iterator<com.yahoo.search.result.Hit> i = hitIterator(result); i.hasNext();) { + com.yahoo.search.result.Hit hit = i.next(); + if ( ! (hit instanceof FastHit)) continue; + FastHit fastHit = (FastHit) hit; + + String traceMsg = "Hit: " + (hitNumber++) + " from " + (fastHit.isCached() ? "cache" : "backend" ); + if ( ! fastHit.isFilled(summaryClass)) + traceMsg += ". Error, hit, not filled"; + query.trace(traceMsg, false, 3); + } + } + } finally { + channel.close(); + } + } + + private static @NonNull Optional<String> quotedSummaryClass(String summaryClass) { + return Optional.of(summaryClass == null ? "[null]" : quote(summaryClass)); + } + + private CacheKey fetchCacheKeyFromHits(HitGroup hits, String summaryClass) { + for (Iterator<Hit> i = hits.unorderedDeepIterator(); i.hasNext();) { + Hit h = i.next(); + if (h instanceof FastHit) { + FastHit hit = (FastHit) h; + if (hit.isFilled(summaryClass)) { + continue; + } + if (hit.getCacheKey() != null) { + return hit.getCacheKey(); + } + } + } + return null; + } + + private Result searchTwoPhase(FS4Channel channel, Query query, QueryPacket queryPacket, CacheKey cacheKey) throws IOException { + + if (isLoggingFine()) + getLogger().finest("sending query packet"); + + try { + boolean couldSend = channel.sendPacket(queryPacket); + if ( ! couldSend) + return new Result(query,ErrorMessage.createBackendCommunicationError("Could not reach '" + getName() + "'")); + } catch (InvalidChannelException e) { + return new Result(query,ErrorMessage.createBackendCommunicationError("Invalid channel " + getName())); + } catch (IllegalStateException e) { + return new Result(query, ErrorMessage.createBackendCommunicationError("Illegal state in FS4: " + e.getMessage())); + } + + BasicPacket[] basicPackets; + + try { + basicPackets = channel.receivePackets(Math.max(50, query.getTimeLeft()), 1); + } catch (ChannelTimeoutException e) { + return new Result(query,ErrorMessage.createTimeout("Timeout while waiting for " + getName())); + } catch (InvalidChannelException e) { + return new Result(query,ErrorMessage.createBackendCommunicationError("Invalid channel for " + getName())); + } + + if (basicPackets.length == 0) { + return new Result(query,ErrorMessage.createBackendCommunicationError(getName() + " got no packets back")); + } + + if (isLoggingFine()) + getLogger().finest("got packets " + basicPackets.length + " packets"); + + ensureInstanceOf(QueryResultPacket.class, basicPackets[0]); + QueryResultPacket resultPacket = (QueryResultPacket) basicPackets[0]; + + checkTimestamp(resultPacket); + + if (isLoggingFine()) + getLogger().finest("got query packet. " + "docsumClass=" + query.getPresentation().getSummary()); + + if (query.getPresentation().getSummary() == null) + query.getPresentation().setSummary(getDefaultDocsumClass()); + + Result result = new Result(query); + + addMetaInfo(query, queryPacket.getQueryPacketData(), resultPacket, result, false); + + addUnfilledHits(result, resultPacket.getDocuments(), false, queryPacket.getQueryPacketData(), cacheKey); + Packet[] packets; + PacketWrapper packetWrapper = cacheControl.lookup(cacheKey, query); + + if (packetWrapper != null) { + cacheControl.updateCacheEntry(cacheKey, query, resultPacket); + } + else { + if (resultPacket.getCoverageFeature() && ! resultPacket.getCoverageFull()) { + // Don't add error here, it was done in first phase + // No check if packetWrapper already exists, since incomplete + // first phase data won't be cached anyway. + } else { + packets = new Packet[1]; + packets[0] = resultPacket; + cacheControl.cache(cacheKey, query, new DocsumPacketKey[0], packets); + } + } + return result; + } + + private Packet[] convertBasicPackets(BasicPacket[] basicPackets) throws ClassCastException { + // trying to cast a BasicPacket[] to Packet[] will compile, + // but lead to a runtime error. At least that's what I got + // from testing and reading the specification. I'm just happy + // if someone tells me what's the proper Java way of doing + // this. -SK + Packet[] packets = new Packet[basicPackets.length]; + + for (int i = 0; i < basicPackets.length; i++) { + packets[i] = (Packet) basicPackets[i]; + } + return packets; + } + + private Packet[] fetchSummaries(FS4Channel channel, Result result, String summaryClass) + throws InvalidChannelException, ChannelTimeoutException, ClassCastException, IOException { + + BasicPacket[] receivedPackets; + boolean summaryNeedsQuery = summaryNeedsQuery(result.getQuery()); + if (result.getQuery().getTraceLevel() >=3) + result.getQuery().trace((summaryNeedsQuery ? "Resending " : "Not resending ") + "query during document summary fetching", 3); + + GetDocSumsPacket docsumsPacket = GetDocSumsPacket.create(result, summaryClass, summaryNeedsQuery); + int compressionLimit = result.getQuery().properties().getInteger(PACKET_COMPRESSION_LIMIT, 0); + docsumsPacket.setCompressionLimit(compressionLimit); + if (compressionLimit != 0) { + docsumsPacket.setCompressionType(result.getQuery().properties().getString(PACKET_COMPRESSION_TYPE, "lz4")); + } + + if (isLoggingFine()) + getLogger().finest("Sending " + docsumsPacket + " on " + channel); + + boolean couldSend = channel.sendPacket(docsumsPacket); + if ( ! couldSend) throw new IOException("Could not successfully send GetDocSumsPacket."); + receivedPackets = channel.receivePackets(Math.max(50, result.getQuery().getTimeLeft()), docsumsPacket.getNumDocsums() + 1); + + if (isLoggingFine()) + getLogger().finest("got " + receivedPackets.length + "docsumPackets"); + + return convertBasicPackets(receivedPackets); + } + + /** + * Returns whether we need to send the query when fetching summaries. + * This is necessary if the query requests summary features or dynamic snippeting + */ + private boolean summaryNeedsQuery(Query query) { + if (query.getRanking().getQueryCache()) return false; // Query is cached in backend + + DocumentDatabase documentDb = getDocumentDatabase(query); + + // Needed to generate a dynamic summary? + DocsumDefinition docsumDefinition = documentDb.getDocsumDefinitionSet().getDocsumDefinition(query.getPresentation().getSummary()); + if (docsumDefinition == null) return true; // stay safe + if (docsumDefinition.isDynamic()) return true; + + // Needed to generate ranking features? + RankProfile rankProfile = documentDb.rankProfiles().get(query.getRanking().getProfile()); + if (rankProfile == null) return true; // stay safe + if (rankProfile.hasSummaryFeatures()) return true; + if (query.getRanking().getListFeatures()) return true; + + // (Don't just add other checks here as there is a return false above) + + return false; + } + + /** + * Whether to mask out the row id from the index uri. + * Masking out the row number is useful when it is necessary to deduplicate + * across rows. That is necessary with searchers which issues several queries + * to produce one result in the first phase, as the grouping searcher - when + * some of those searchers go to different rows, a mechanism is needed to detect + * duplicates returned from different rows before the summary is requested. + * Producing an index id which is the same across rows and using that as the + * hit uri provides this. Note that this only works if the document ids are the + * same for all the nodes (rows) in a column. This is usually the case for + * batch and incremental indexing, but not for realtime. + */ + + public long getEditionTimeStamp() { + return editionTimeStamp; + } + + public void setEditionTimeStamp(long editionTime) { + this.editionTimeStamp = editionTime; + } + + public String toString() { + return "fast searcher (" + getName() + ") " + backend; + } + + /** + * Returns an array of the hits contained in this result + * + * @param filled true to return all hits, false to return only unfilled hits + * @return array of docids, empty array if no hits + */ + private DocsumPacketKey[] getPacketKeys(Result result, String summaryClass, boolean filled) { + DocsumPacketKey[] packetKeys = new DocsumPacketKey[result.getHitCount()]; + int x = 0; + + for (Iterator<com.yahoo.search.result.Hit> i = hitIterator(result); i.hasNext();) { + com.yahoo.search.result.Hit hit = i.next(); + if (hit instanceof FastHit) { + FastHit fastHit = (FastHit) hit; + if(filled || !fastHit.isFilled(summaryClass)) { + packetKeys[x] = new DocsumPacketKey(fastHit.getGlobalId(), fastHit.getPartId(), summaryClass); + x++; + } + } + } + if (x < packetKeys.length) { + DocsumPacketKey[] tmp = new DocsumPacketKey[x]; + + System.arraycopy(packetKeys, 0, tmp, 0, x); + return tmp; + } else { + return packetKeys; + } + } + + protected boolean isLoggingFine() { + return getLogger().isLoggable(Level.FINE); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java new file mode 100644 index 00000000000..b622f5c62c5 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; +import com.yahoo.container.search.LegacyEmulationConfig; +import com.yahoo.search.result.FeatureData; + +/** + * Class representing a "feature data" field. This was historically + * just a string containing JSON; now it's a structure of + * data (that will be rendered as JSON by default). + */ +public class FeatureDataField extends LongstringField { + + public FeatureDataField (String name) { + super(name); + } + + @Override + public String toString() { + return "field " + getName() + " type FeatureDataField"; + } + + public Object convert(Inspector value) { + if (! value.valid()) { + if (getEmulConfig().stringBackedFeatureData()) { + return ""; + } else if (getEmulConfig().forceFillEmptyFields()) { + return new FeatureData(com.yahoo.data.access.simple.Value.empty()); + } else { + return null; + } + } + if (value.type() == Type.STRING) { + return value.asString(); + } + FeatureData obj = new FeatureData(value); + if (getEmulConfig().stringBackedFeatureData()) { + return obj.toJson(); + } else { + return obj; + } + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java new file mode 100644 index 00000000000..ed5c7edd4da --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.search.result.NanNumber; +import com.yahoo.data.access.Inspector; + + +/** + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00f8lster Lidal</a> + */ +public class FloatField extends DocsumField { + static final double EMPTY_VALUE = Float.NaN; + + public FloatField(String name) { + super(name); + } + + private Object convert(float value) { + if (Float.isNaN(value)) { + return NanNumber.NaN; + } else { + return Float.valueOf(value); + } + } + + public Object decode(ByteBuffer b) { + return convert(b.getFloat()); + } + + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + public int getLength(ByteBuffer b) { + int offset = b.position(); + final int bytelength = Float.SIZE >> 3; + b.position(offset + bytelength); + return bytelength; + } + + public Object convert(Inspector value) { + return convert((float)value.asDouble(EMPTY_VALUE)); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/GroupingListHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/GroupingListHit.java new file mode 100644 index 00000000000..f8425ba8cfd --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/GroupingListHit.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import java.util.List; + +import com.yahoo.fs4.QueryPacketData; +import com.yahoo.search.result.Hit; +import com.yahoo.searchlib.aggregation.Grouping; + +// TODO: Author! +public class GroupingListHit extends Hit { + private static final long serialVersionUID = -6645125887873082234L; + + /** for unit tests only, may give problems if grouping contains docsums */ + public GroupingListHit(List<Grouping> groupingList) { + this(groupingList, null); + } + + public GroupingListHit(List<Grouping> groupingList, + DocsumDefinitionSet defs) + { + super("meta:grouping", 0); + this.groupingList = groupingList; + this.defs = defs; + } + public boolean isMeta() { return true; } + + public List<Grouping> getGroupingList() { return groupingList; } + public DocsumDefinitionSet getDocsumDefinitionSet() { return defs; } + + private final List<Grouping> groupingList; + private final DocsumDefinitionSet defs; + private QueryPacketData queryPacketData; + + public void setQueryPacketData(QueryPacketData queryPacketData) { + this.queryPacketData = queryPacketData; + } + + /** Returns encoded query data from the query used to create this, or null if none present */ + public QueryPacketData getQueryPacketData() { + return queryPacketData; + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/Int64Field.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Int64Field.java new file mode 100644 index 00000000000..2759f313d52 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Int64Field.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a integer field in the result set + * + */ +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.search.result.NanNumber; +import com.yahoo.data.access.Inspector; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ +public class Int64Field extends DocsumField { + static final long EMPTY_VALUE = Long.MIN_VALUE; + + public Int64Field(String name) { + super(name); + } + + private Object convert(long value) { + if (value == EMPTY_VALUE) { + return NanNumber.NaN; + } else { + return Long.valueOf(value); + } + } + + public Object decode(ByteBuffer b) { + return convert(b.getLong()); + } + + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + public String toString() { + return "field " + getName() + " type int64"; + } + + public int getLength(ByteBuffer b) { + int offset = b.position(); + final int bytelength = Long.SIZE >> 3; + b.position(offset + bytelength); + return bytelength; + } + + public Object convert(Inspector value) { + return convert(value.asLong(EMPTY_VALUE)); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java new file mode 100644 index 00000000000..b134ea49bac --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a integer field in the result set + * + */ +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.search.result.NanNumber; +import com.yahoo.data.access.Inspector; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ +public class IntegerField extends DocsumField { + static final int EMPTY_VALUE = Integer.MIN_VALUE; + + public IntegerField(String name) { + super(name); + } + + private Object convert(int value) { + if (value == EMPTY_VALUE) { + return NanNumber.NaN; + } else { + return Integer.valueOf(value); + } + } + + public Object decode(ByteBuffer b) { + return convert(b.getInt()); + } + + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + public String toString() { + return "field " + getName() + " type int"; + } + + public int getLength(ByteBuffer b) { + int offset = b.position(); + final int bytelength = Integer.SIZE >> 3; + b.position(offset + bytelength); + return bytelength; + } + + public Object convert(Inspector value) { + return convert((int)value.asLong(EMPTY_VALUE)); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java new file mode 100644 index 00000000000..d61a15723ac --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java @@ -0,0 +1,180 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.io.SlowInflate; +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.text.Utf8; +import com.yahoo.data.access.*; +import com.yahoo.data.access.simple.Value; + + +/** + * Class representing a JSON string field in the result set + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class JSONField extends DocsumField implements VariableLengthField { + public JSONField(String name) { + super(name); + } + + @Override + public Object decode(ByteBuffer b) { + long dataLen = 0; + long len = ((long) b.getInt()) & 0xffffffffL; + boolean compressed; + JSONString field; + + // if MSB is set this is a compressed field. set the compressed + // flag accordingly and decompress the data + compressed = ((len & 0x80000000) != 0); + if (compressed) { + len &= 0x7fffffff; + dataLen = b.getInt(); + len -= 4; + } + + byte[] tmp = new byte[(int) len]; + + b.get(tmp); + + if (compressed) { + SlowInflate inf = new SlowInflate(); + + tmp = inf.unpack(tmp, (int) dataLen); + } + + field = new JSONString(Utf8.toString(tmp)); + return field; + } + + @Override + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + @Override + public String toString() { + return "field " + getName() + " type JSONString"; + } + + @Override + public int getLength(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int len = b.getInt() & 0x7fffffff; + b.position(offset + len + (Integer.SIZE >> 3)); + return len + (Integer.SIZE >> 3); + } + + @Override + public boolean isCompressed(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int compressed = b.getInt() & 0x80000000; + b.position(offset); + return compressed != 0; + } + + @Override + public int sizeOfLength() { + return Integer.SIZE >> 3; + } + + private static class CompatibilityConverter { + Value.ArrayValue target = new Value.ArrayValue(); + + Inspector stringify(Inspector value) { + if (value.type() == Type.STRING) return value; + if (value.type() == Type.LONG) { + String str = String.valueOf(value.asLong()); + return new Value.StringValue(str); + } + if (value.type() == Type.DOUBLE) { + String str = String.valueOf(value.asDouble()); + return new Value.StringValue(str); + } + String str = value.toString(); + return new Value.StringValue(str); + } + } + + private static class ArrConv extends CompatibilityConverter + implements ArrayTraverser + { + @Override + public void entry(int idx, Inspector value) { + target.add(stringify(value)); + } + } + + private static class WsConv1 extends CompatibilityConverter + implements ArrayTraverser + { + @Override + public void entry(int idx, Inspector value) { + Value.ArrayValue obj = new Value.ArrayValue(); + obj.add(stringify(value.entry(0))); + obj.add(value.entry(1)); + target.add(obj); + } + } + + private static class WsConv2 extends CompatibilityConverter + implements ArrayTraverser + { + @Override + public void entry(int idx, Inspector value) { + Value.ArrayValue obj = new Value.ArrayValue(); + obj.add(stringify(value.field("item"))); + obj.add(value.field("weight")); + target.add(obj); + } + } + + static Inspector convertTop(Inspector value) { + if (value.type() == Type.ARRAY && value.entryCount() > 0) { + Inspector first = value.entry(0); + if (first.type() == Type.ARRAY && first.entryCount() == 2) { + // old style weighted set + WsConv1 conv = new WsConv1(); + value.traverse(conv); + return conv.target; + } + if (first.type() == Type.OBJECT && + first.fieldCount() == 2 && + first.field("item").valid() && + first.field("weight").valid()) + { + // new style weighted set + WsConv2 conv = new WsConv2(); + value.traverse(conv); + return conv.target; + } + if (first.type() == Type.LONG) { + ArrConv conv = new ArrConv(); + value.traverse(conv); + return conv.target; + } + if (first.type() == Type.DOUBLE) { + ArrConv conv = new ArrConv(); + value.traverse(conv); + return conv.target; + } + } + return value; + } + + public Object convert(Inspector value) { + if (value.valid()) { + return new JSONString(convertTop(value)); + } else { + return new JSONString(""); + } + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java new file mode 100644 index 00000000000..617f382f462 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a long data field in the result set. + * + */ +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.io.SlowInflate; +import com.yahoo.prelude.hitfield.RawData; +import com.yahoo.data.access.simple.Value; +import com.yahoo.data.access.Inspector; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ +public class LongdataField extends DocsumField implements VariableLengthField { + public LongdataField(String name) { + super(name); + } + + private Object convert(byte[] value) { + return new RawData(value); + } + + @Override + public Object decode(ByteBuffer b) { + long dataLen = 0; + long len = ((long) b.getInt()) & 0xffffffffL; + boolean compressed; + + // if MSB is set this is a compressed field. set the compressed + // flag accordingly and decompress the data + compressed = ((len & 0x80000000) != 0); + if (compressed) { + len &= 0x7fffffff; + dataLen = b.getInt(); + len -= 4; + } + + byte[] tmp = new byte[(int) len]; + + b.get(tmp); + + if (compressed) { + SlowInflate inf = new SlowInflate(); + + tmp = inf.unpack(tmp, (int) dataLen); + } + return convert(tmp); + } + + @Override + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + @Override + public int getLength(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int len = b.getInt() & 0x7fffffff; + b.position(offset + len + (Integer.SIZE >> 3)); + return len + (Integer.SIZE >> 3); + } + + @Override + public boolean isCompressed(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int compressed = b.getInt() & 0x80000000; + b.position(offset); + return compressed != 0; + } + + @Override + public int sizeOfLength() { + return Integer.SIZE >> 3; + } + + @Override + public Object convert(Inspector value) { + return convert(value.asData(Value.empty().asData())); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java new file mode 100644 index 00000000000..744476beaa5 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a long string field in the result set. + * + */ +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.io.SlowInflate; +import com.yahoo.text.Utf8; +import com.yahoo.data.access.Inspector; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ +public class LongstringField extends DocsumField implements VariableLengthField { + public LongstringField(String name) { + super(name); + } + + @Override + public Object decode(ByteBuffer b) { + long dataLen = 0; + long len = ((long) b.getInt()) & 0xffffffffL; + boolean compressed; + String field; + + // if MSB is set this is a compressed field. set the compressed + // flag accordingly and decompress the data + compressed = ((len & 0x80000000) != 0); + if (compressed) { + len &= 0x7fffffff; + dataLen = b.getInt(); + len -= 4; + } + + byte[] tmp = new byte[(int) len]; + + b.get(tmp); + + if (compressed) { + SlowInflate inf = new SlowInflate(); + + tmp = inf.unpack(tmp, (int) dataLen); + } + field = Utf8.toString(tmp); + return field; + } + + @Override + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + @Override + public int getLength(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int len = b.getInt() & 0x7fffffff; + b.position(offset + len + (Integer.SIZE >> 3)); + return len + (Integer.SIZE >> 3); + } + + @Override + public boolean isCompressed(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int compressed = b.getInt() & 0x80000000; + b.position(offset); + return compressed != 0; + } + + @Override + public int sizeOfLength() { + return Integer.SIZE >> 3; + } + + @Override + public Object convert(Inspector value) { + return value.asString(""); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/PacketCache.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/PacketCache.java new file mode 100644 index 00000000000..e5a7d433324 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/PacketCache.java @@ -0,0 +1,189 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.logging.Logger; + +import com.yahoo.log.LogLevel; + + +/** + * An LRU cache using number of hits cached inside the results as + * size limiting factor. Directly modelled after com.yahoo.collections.Cache. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class PacketCache extends LinkedHashMap<CacheKey, PacketWrapper> { + + /** + * + */ + private static final long serialVersionUID = -7403077211906108356L; + + /** The <i>current</i> number of bytes of packets in this cache */ + private int totalSize; + + /** The maximum number of bytes of packets in this cache */ + private final int capacity; + + /** The max size of a cached item compared to the total size */ + private int maxCacheItemPercentage = 1; + + /** The max age for a valid cache entry, 0 mean infinite */ + private final long maxAge; + + private static final Logger log = Logger.getLogger(PacketCache.class.getName()); + + public void clear() { + super.clear(); + totalSize = 0; + } + + /** + * Sets the max size of a cached item compared to the total size + * Cache requests for larger objects will be ignored + */ + public void setMaxCacheItemPercentage(int maxCapacityPercentage) { + maxCacheItemPercentage = maxCapacityPercentage; + } + + /** + * Creates a cache with a size given by + * cachesizemegabytes*2^20+cachesizebytes + * + * @param capacityMegaBytes the cache size, measured in megabytes + * @param capacityBytes additional number of bytes to add to the cache size + * @param maxAge seconds a cache entry is valid, 0 or less are illegal arguments + */ + public PacketCache(int capacityMegaBytes,int capacityBytes,double maxAge) { + // hardcoded inital entry capacity, won't matter much anyway + // after a while + super(12500, 1.0f, true); + if (maxAge <= 0.0d) { + throw new IllegalArgumentException("maxAge <= 0 not legal on 5.1, use some very large number for no timeout."); + } + if (capacityMegaBytes > (Integer.MAX_VALUE >> 20)) { + log.log(LogLevel.INFO, "Packet cache of more than 2 GB requested. Reverting to 2 GB packet cache."); + this.capacity = Integer.MAX_VALUE; + } else { + this.capacity = (capacityMegaBytes << 20) + capacityBytes; + } + if (this.capacity <= 0) { + throw new IllegalArgumentException("Total cache size set to 0 or less bytes. If no caching is desired, avoid creating this object instead."); + } + this.maxAge = (long) (maxAge * 1000.0d); + } + + /** + * Overrides LinkedHashMap.removeEldestEntry as suggested to implement LRU cache. + */ + protected boolean removeEldestEntry(Map.Entry<CacheKey, PacketWrapper> eldest) + { + if (totalSize > capacity) { + totalSize -= eldest.getValue().getPacketsSize(); + return true; + } + return false; + } + + private void removeOverflow() { + if (totalSize < capacity) return; + + for (Iterator<PacketWrapper> i = values().iterator(); i.hasNext();) { + PacketWrapper eldestEntry = i.next(); + totalSize -= eldestEntry.getPacketsSize(); + + i.remove(); + if (totalSize < capacity) { + break; + } + } + } + + public int getCapacity() { + return capacity >> 20; + } + + public int getByteCapacity() { + return capacity; + } + + /** + * Adds a PacketWrapper object to this cache, + * unless the size is more than maxCacheItemPercentage of the total size + */ + public PacketWrapper put(CacheKey key, PacketWrapper value) { + return put(key, value, System.currentTimeMillis()); + } + + /** + * Adds a BasicPacket array to this cache, + * unless the size is more than maxCacheItemPercentage of the total size + * + * @param timestamp the timestamp for the first packet in the array, + * unit milliseconds + */ + public PacketWrapper put(CacheKey key, PacketWrapper result, long timestamp) { + int size = result.getPacketsSize(); + + if (size > 0) { + result.setTimestamp(timestamp); + } + + // don't insert if it is too big + if (size * 100 > capacity * maxCacheItemPercentage) { + // removeField the old one since that is now stale. + return remove(key); + } + + totalSize += size; + PacketWrapper previous = super.put(key, result); + if (previous != null) { + totalSize -= previous.getPacketsSize(); + } + if (totalSize > (capacity * 1.1)) { + removeOverflow(); + } + + return previous; + } + + public PacketWrapper get(CacheKey key) { + return get(key, System.currentTimeMillis()); + } + + public PacketWrapper get(CacheKey key, long now) { + PacketWrapper result = super.get(key); + + if (result == null) { + return result; + } + + long timestamp = result.getTimestamp(); + + if ((now - timestamp) > maxAge) { + remove(key); + return null; + } else { + return result; + } + } + + public PacketWrapper remove(CacheKey key) { + PacketWrapper removed = super.remove(key); + + if (removed != null) { + totalSize -= removed.getPacketsSize(); + } + return removed; + } + + public int totalPacketSize() { + return totalSize; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/PacketWrapper.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/PacketWrapper.java new file mode 100644 index 00000000000..1cc9678843c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/PacketWrapper.java @@ -0,0 +1,300 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import java.util.*; +import java.util.logging.Logger; + +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.DocsumPacket; +import com.yahoo.fs4.DocumentInfo; +import com.yahoo.fs4.Packet; +import com.yahoo.fs4.QueryResultPacket; +import com.yahoo.document.GlobalId; +import com.yahoo.document.DocumentId; + + +/** + * A wrapper for cache entries to make it possible to check whether the + * hits are truly correct. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Lidal</a> + */ +public class PacketWrapper implements Cloneable { + private static Logger log = Logger.getLogger(PacketWrapper.class.getName()); + + final int keySize; + // associated result packets, sorted in regard to offset + private ArrayList<BasicPacket> resultPackets = new ArrayList<>(3); // length = "some small number" + LinkedHashMap<DocsumPacketKey, BasicPacket> packets; + + private static class ResultPacketComparator<T extends BasicPacket> implements Comparator<T> { + @Override + public int compare(T o1, T o2) { + QueryResultPacket r1 = (QueryResultPacket) o1; + QueryResultPacket r2 = (QueryResultPacket) o2; + return r1.getOffset() - r2.getOffset(); + } + } + + private static ResultPacketComparator<BasicPacket> resultPacketComparator = new ResultPacketComparator<>(); + + public PacketWrapper(CacheKey key, DocsumPacketKey[] packetKeys, BasicPacket[] bpackets) { + // Should not support key == null + this.keySize = key.byteSize(); + resultPackets.add(bpackets[0]); + this.packets = new LinkedHashMap<>(); + Packet[] ppackets = new Packet[packetKeys.length]; + + for (int i = 0; i < packetKeys.length; i++) { + ppackets[i] = (Packet) bpackets[i + 1]; + } + addDocsums(packetKeys, ppackets); + } + + /** + * Only used by PacketCacheTestCase, should not be used otherwise + */ + public PacketWrapper(CacheKey key, BasicPacket[] packets) { + // Should support key == null as this is for testing + if (key == null) { + keySize = 0; + } else { + this.keySize = key.byteSize(); + } + resultPackets.add(packets[0]); + this.packets = new LinkedHashMap<>(); + for (int i = 0; i < packets.length - 1; i++) { + this.packets.put(new DocsumPacketKey(new GlobalId(new DocumentId("doc:test:" + i).getGlobalId()), i, null), packets[i + 1]); + } + + } + + public QueryResultPacket getFirstResultPacket() { + if (resultPackets.size() > 0) { + return (QueryResultPacket) resultPackets.get(0); + } else { + return null; + } + } + + /** + * @return list of documents, null if not all are available + */ + public List<DocumentInfo> getDocuments(int offset, int hits) { + // speculatively allocate list for the hits + List<DocumentInfo> docs = new ArrayList<>(hits); + int currentOffset = 0; + QueryResultPacket r = getFirstResultPacket(); + if (offset >= r.getTotalDocumentCount()) { + // shortcut especially for results with 0 hits + // >= both necessary for end of result sets and + // offset == 0 && totalDocumentCount == 0 + return docs; + } + for (Iterator<BasicPacket> i = resultPackets.iterator(); i.hasNext();) { + QueryResultPacket result = (QueryResultPacket) i.next(); + if (result.getOffset() > offset + currentOffset) { + // we haven't got all the requested document info objects + return null; + } + if (result.getOffset() + result.getDocumentCount() + <= currentOffset + offset) { + // no new hits available + continue; + } + List<DocumentInfo> documents = result.getDocuments(); + int packetOffset = (offset + currentOffset) - result.getOffset(); + int afterLastDoc = Math.min(documents.size(), packetOffset + hits); + for (Iterator<DocumentInfo> j = documents.subList(packetOffset, afterLastDoc).iterator(); + docs.size() < hits && j.hasNext(); + ++currentOffset) { + docs.add(j.next()); + } + if (hits == docs.size() + || offset + docs.size() >= result.getTotalDocumentCount()) { + // We have the hits we need, or there are no more hits available + return docs; + } + } + return null; + } + + public void addResultPacket(QueryResultPacket resultPacket) { + // This function only keeps the internal list sorted according + // to offset + int insertionPoint; + QueryResultPacket r; + + if (resultPacket.getDocumentCount() == 0) { + return; // do not add a packet which does not contain new info + } + + insertionPoint = Collections.binarySearch(resultPackets, + resultPacket, + resultPacketComparator); + if (insertionPoint < 0) { + // new offset + insertionPoint = ~insertionPoint; // (insertionPoint + 1) * -1; + resultPackets.add(insertionPoint, resultPacket); + cleanResultPackets(); + } else { + // there exists a packet with same offset + r = (QueryResultPacket) resultPackets.get(insertionPoint); + if (resultPacket.getDocumentCount() > r.getDocumentCount()) { + resultPackets.set(insertionPoint, resultPacket); + cleanResultPackets(); + } + } + } + + private void cleanResultPackets() { + int marker; + QueryResultPacket previous; + if (resultPackets.size() == 1) { + return; + } + + // we know the list is sorted with regard to offset + // First ensure the list grows in regards to lastOffset as well. + // Could have done this addResultPacket, but this makes the code + // simpler. + previous = (QueryResultPacket) resultPackets.get(0); + for (int i = 1; i < resultPackets.size(); ++i) { + QueryResultPacket r = (QueryResultPacket) resultPackets.get(i); + if (r.getOffset() + r.getDocumentCount() + <= previous.getOffset() + previous.getDocumentCount()) { + resultPackets.remove(i--); + } else { + previous = r; + } + } + + marker = 0; + while (marker < (resultPackets.size() - 2)) { + QueryResultPacket r0 = (QueryResultPacket) resultPackets.get(marker); + QueryResultPacket r1 = (QueryResultPacket) resultPackets.get(marker + 1); + QueryResultPacket r2 = (QueryResultPacket) resultPackets.get(marker + 2); + int nextOffset = r0.getOffset() + r0.getDocumentCount(); + + if (r1.getOffset() < nextOffset + && r2.getOffset() <= nextOffset) { + resultPackets.remove(marker + 1); + } + ++marker; + } + } + + /** Only for testing. */ + public List<BasicPacket> getResultPackets() { + return resultPackets; + } + + public void addDocsums(DocsumPacketKey[] packetKeys, BasicPacket[] bpackets, + int offset) { + Packet[] ppackets = new Packet[packetKeys.length]; + + for (int i = 0; i < packetKeys.length; i++) { + ppackets[i] = (Packet) bpackets[i + offset]; + } + addDocsums(packetKeys, ppackets); + } + + public void addDocsums(DocsumPacketKey[] packetKeys, Packet[] packets) { + if (packetKeys == null || packets == null) { + log.warning( + "addDocsums called with " + + (packetKeys == null ? "packetKeys == null " : "") + + (packets == null ? "packets == null" : "")); + return; + } + for (int i = 0; i < packetKeys.length && i < packets.length; i++) { + if (packetKeys[i] == null) { + log.warning( + "addDocsums called, but packetsKeys[" + i + "] is null"); + } else if (packets[i] instanceof DocsumPacket) { + DocsumPacket dp = (DocsumPacket) packets[i]; + + if (packetKeys[i].getGlobalId().equals(dp.getGlobalId()) + && dp.getData().length > 0) + { + this.packets.put(packetKeys[i], packets[i]); + log.fine("addDocsums " + i + " globalId: " + dp.getGlobalId()); + } else { + log.warning("not caching bad Docsum for globalId " + packetKeys[i].getGlobalId() + ": " + dp); + } + } else { + log.warning( + "addDocsums called, but packets[" + i + + "] is not a DocsumPacket instance"); + } + } + } + + public int getNumPackets() { + return packets.size(); + } + + BasicPacket getPacket(GlobalId globalId, int partid, String summaryClass) { + return getPacket( + new DocsumPacketKey(globalId, partid, summaryClass)); + } + + BasicPacket getPacket(DocsumPacketKey packetKey) { + return packets.get(packetKey); + } + + long getTimestamp() { + return getFirstResultPacket().getTimestamp(); + } + + public void setTimestamp(long timestamp) { + getFirstResultPacket().setTimestamp(timestamp); + } + + public int getPacketsSize() { + int size = 0; + + for (Iterator<BasicPacket> i = resultPackets.iterator(); i.hasNext();) { + QueryResultPacket r = (QueryResultPacket) i.next(); + int l = r.getLength(); + + if (l < 0) { + log.warning("resultpacket length " + l); + l = 10240; + } + size += l; + } + for (Iterator<BasicPacket> i = packets.values().iterator(); i.hasNext();) { + BasicPacket packet = i.next(); + int l = packet.getLength(); + + if (l < 0) { + log.warning("BasicPacket length " + l); + l = 10240; + } + size += l; + } + size += keySize; + return size; + } + + /** + * Straightforward shallow copy. + */ + @SuppressWarnings("unchecked") + public Object clone() { + try { + PacketWrapper other = (PacketWrapper) super.clone(); + other.resultPackets = (ArrayList<BasicPacket>) resultPackets.clone(); + if (packets != null) { + other.packets = (LinkedHashMap<DocsumPacketKey, BasicPacket>) packets.clone(); + } + return other; + } catch (CloneNotSupportedException e) { + throw new RuntimeException("A non-cloneable superclass has been inserted.", + e); + } + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/RankProfile.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/RankProfile.java new file mode 100644 index 00000000000..66931f37369 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/RankProfile.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +/** + * Information about a rank profile + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +class RankProfile { + + private final String name; + + private final boolean hasSummaryFeatures; + + private final boolean hasRankFeatures; + + public RankProfile(String name, boolean hasSummaryFeatures, boolean hasRankFeatures) { + this.name = name; + this.hasSummaryFeatures = hasSummaryFeatures; + this.hasRankFeatures = hasRankFeatures; + } + + public String getName() { return name; } + + /** Returns true if this rank profile has summary features */ + public boolean hasSummaryFeatures() { return hasSummaryFeatures; } + + /** Returns true if this rank profile has rank features */ + public boolean hasRankFeatures() { return hasRankFeatures; } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java new file mode 100644 index 00000000000..e9c19590102 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a short field in the result set + * + */ +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.search.result.NanNumber; +import com.yahoo.data.access.Inspector; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ + +public class ShortField extends DocsumField { + static final short EMPTY_VALUE = Short.MIN_VALUE; + + public ShortField(String name) { + super(name); + } + + private Object convert(short value) { + if (value == EMPTY_VALUE) { + return NanNumber.NaN; + } else { + return Short.valueOf(value); + } + } + + public Object decode(ByteBuffer b) { + return convert(b.getShort()); + } + + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + public int getLength(ByteBuffer b) { + int offset = b.position(); + final int bytelength = Short.SIZE >> 3; + b.position(offset + bytelength); + return bytelength; + } + + public Object convert(Inspector value) { + return convert((short)value.asLong(EMPTY_VALUE)); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java new file mode 100644 index 00000000000..671188e4cae --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a string field in the result set + * + */ +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.text.Utf8; +import com.yahoo.data.access.Inspector; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + */ +public class StringField extends DocsumField implements VariableLengthField { + public StringField(String name) { + super(name); + } + + @Override + public Object decode(ByteBuffer b) { + int length = ((int) b.getShort()) & 0xffff; + Object field; + + field = Utf8.toString(b.array(), b.arrayOffset() + b.position(), length); + b.position(b.position() + length); + return field; + } + + @Override + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + @Override + public String toString() { + return "field " + getName() + " type string"; + } + + @Override + public int getLength(ByteBuffer b) { + int offset = b.position(); + int len = ((int) b.getShort()) & 0xffff; + b.position(offset + len + (Short.SIZE >> 3)); + return len + (Short.SIZE >> 3); + } + + @Override + public int sizeOfLength() { + return Short.SIZE >> 3; + } + + @Override + public Object convert(Inspector value) { + return value.asString(""); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java new file mode 100644 index 00000000000..f0f4b82c22a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import com.yahoo.search.result.StructuredData; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; +import com.yahoo.container.search.LegacyEmulationConfig; +import com.yahoo.prelude.hitfield.JSONString; + +/** + * Class representing a XML rendered structured data field in the result set + */ +public class StructDataField extends JSONField { + + public StructDataField(String name) { + super(name); + } + + @Override + public String toString() { + return "field " + getName() + " type StructDataField"; + } + + public Object convert(Inspector value) { + if (getEmulConfig().stringBackedStructuredData() || + value.type() == Type.STRING) + { + return super.convert(value); + } + return new StructuredData(value); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java new file mode 100644 index 00000000000..97a711d8590 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SummaryParameters.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +/** + * Wrapper for document summary parameters and configuration. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class SummaryParameters { + + public final String defaultClass; + + public SummaryParameters(String defaultClass) { + if (defaultClass != null && defaultClass.isEmpty()) + this.defaultClass = null; + else + this.defaultClass = defaultClass; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/TimeoutException.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TimeoutException.java new file mode 100644 index 00000000000..8c3d587a059 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TimeoutException.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import java.io.IOException; + +/** + * Thrown on communication timeouts + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +@SuppressWarnings("serial") +public class TimeoutException extends IOException { + + public TimeoutException(String message) { + super(message); + } +} + diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java new file mode 100644 index 00000000000..f169533f8db --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +/** + * Interface to easier find the start of the actual data for variable length + * fields. + * + * @author <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public interface VariableLengthField { + public int sizeOfLength(); +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java new file mode 100644 index 00000000000..820c764de06 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -0,0 +1,653 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import java.util.Optional; +import com.yahoo.collections.TinyIdentitySet; +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.DocsumPacket; +import com.yahoo.fs4.DocumentInfo; +import com.yahoo.fs4.ErrorPacket; +import com.yahoo.fs4.QueryPacketData; +import com.yahoo.fs4.Packet; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.fs4.QueryResultPacket; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.io.HexDump; +import com.yahoo.log.LogLevel; +import com.yahoo.prelude.ConfigurationException; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; +import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.protect.Validator; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.cluster.PingableSearcher; +import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.vespa.objects.BufferSerializer; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; + + +/** + * Superclass for backend searchers. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +@SuppressWarnings("deprecation") +public abstract class VespaBackEndSearcher extends PingableSearcher { + + private static final CompoundName grouping=new CompoundName("grouping"); + private static final CompoundName combinerows=new CompoundName("combinerows"); + protected static final CompoundName PACKET_COMPRESSION_LIMIT = new CompoundName("packetcompressionlimit"); + protected static final CompoundName PACKET_COMPRESSION_TYPE = new CompoundName("packetcompressiontype"); + protected static final CompoundName TRACE_DISABLE = new CompoundName("trace.disable"); + + /** The set of all document databases available in the backend handled by this searcher */ + private Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); + private DocumentDatabase defaultDocumentDb = null; + + /** Default docsum class. null means "unset" and is the default value */ + private String defaultDocsumClass = null; + + /** Returns an iterator which returns all hits below this result **/ + protected Iterator<Hit> hitIterator(Result result) { + return result.hits().unorderedDeepIterator(); + } + + private boolean localDispatching = true; + + /** The name of this source */ + private String name; + + /** Cache wrapper */ + protected CacheControl cacheControl = null; + /** + * The number of last significant bits in the partId which specifies the + * row number in this backend, + * the rest specifies the column. 0 if not known. + */ + private int rowBits = 0; + /** Searchcluster number */ + private int sourceNumber; + + protected final String getName() { return name; } + protected final String getDefaultDocsumClass() { return defaultDocsumClass; } + + /** Sets default document summary class. Default is null */ + private void setDefaultDocsumClass(String docsumClass) { defaultDocsumClass = docsumClass; } + + /** Returns the packet cache controller of this */ + public final CacheControl getCacheControl() { return cacheControl; } + + /** + * Searches a search cluster + * This is an endpoint - searchers will never propagate the search to any nested searcher. + * + * @param query the query to search + * @param queryPacket the serialized query representation to pass to the search cluster + * @param cacheKey the cache key created from the query packet, or null if caching is not used + * @param execution the query execution context + */ + protected abstract Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution); + + protected abstract void doPartialFill(Result result, String summaryClass); + + private Result cacheLookupFirstPhase(CacheKey key, QueryPacketData queryPacketData, Query query, int offset, int hits, String summaryClass) throws IOException { + PacketWrapper packetWrapper = cacheControl.lookup(key, query); + + if (packetWrapper == null) return null; + + // Check if the cache entry contains the requested hits + List<DocumentInfo> documents = packetWrapper.getDocuments(offset, hits); + if (documents == null) return null; + + if (query.getPresentation().getSummary() == null) + query.getPresentation().setSummary(getDefaultDocsumClass()); + Result result = new Result(query); + QueryResultPacket resultPacket = packetWrapper.getFirstResultPacket(); + + addMetaInfo(query, queryPacketData, resultPacket, result, true); + if (packetWrapper.getNumPackets() == 0) + addUnfilledHits(result, documents, true, queryPacketData, key); + else + addCachedHits(result, packetWrapper, summaryClass, documents); + return result; + } + + + protected DocumentDatabase getDocumentDatabase(Query query) { + if (query.getModel().getRestrict().size() == 1) { + String docTypeName = (String)query.getModel().getRestrict().toArray()[0]; + DocumentDatabase db = documentDbs.get(docTypeName); + if (db != null) { + return db; + } + } + return defaultDocumentDb; + } + + private void resolveDocumentDatabase(Query query) { + DocumentDatabase docDb = getDocumentDatabase(query); + if (docDb != null) { + query.getModel().setDocumentDb(docDb.getName()); + } + } + + public final void init(SummaryParameters docSumParams, ClusterParams clusterParams, CacheParams cacheParams, + DocumentdbInfoConfig documentdbInfoConfig) { + this.name = clusterParams.searcherName; + this.sourceNumber = clusterParams.clusterNumber; + this.rowBits = clusterParams.rowBits; + + Validator.ensureNotNull("Name of Vespa backend integration", getName()); + + setDefaultDocsumClass(docSumParams.defaultClass); + + if (documentdbInfoConfig != null) { + for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { + DocumentDatabase db = new DocumentDatabase(docDb, clusterParams.emulation); + if (documentDbs.isEmpty()) { + defaultDocumentDb = db; + } + documentDbs.put(docDb.name(), db); + } + } + + if (cacheParams.cacheControl == null) { + this.cacheControl = new CacheControl(cacheParams.cacheMegaBytes, cacheParams.cacheTimeOutSeconds); + } else { + this.cacheControl = cacheParams.cacheControl; + } + } + + protected void transformQuery(Query query) { } + + public Result search(Query query, Execution execution) { + // query root should not be null here + Item root = query.getModel().getQueryTree().getRoot(); + if (root == null || root instanceof NullItem) { + return new Result(query, ErrorMessage.createNullQuery(query.getHttpRequest().getUri().toString())); + } + + QueryRewrite.optimizeByRestrict(query); + QueryRewrite.optimizeAndNot(query); + QueryRewrite.collapseSingleComposites(query); + + root = query.getModel().getQueryTree().getRoot(); + if (root == null || root instanceof NullItem) // root can become null after optimization + return new Result(query); + + resolveDocumentDatabase(query); + transformQuery(query); + traceQuery(name, "search", query, query.getOffset(), query.getHits(), 1, Optional.<String>empty()); + + root = query.getModel().getQueryTree().getRoot(); + if (root == null || root instanceof NullItem) // root can become null after resolving and transformation? + return new Result(query); + + QueryPacket queryPacket = QueryPacket.create(query); + int compressionLimit = query.properties().getInteger(PACKET_COMPRESSION_LIMIT, 0); + queryPacket.setCompressionLimit(compressionLimit); + if (compressionLimit != 0) { + queryPacket.setCompressionType(query.properties().getString(PACKET_COMPRESSION_TYPE, "lz4")); + } + + if (isLoggingFine()) + getLogger().fine("made QueryPacket: " + queryPacket); + + Result result = null; + CacheKey cacheKey = null; + if (cacheControl.useCache(query)) { + cacheKey = new CacheKey(queryPacket); + result = getCached(cacheKey, queryPacket.getQueryPacketData(), query); + } + + if (result == null) { + String next = null; + result = doSearch2(query, queryPacket, cacheKey, execution); + if (isLoggingFine()) { + getLogger().fine("Result NOT retrieved from cache"); + } + + if (query.getTraceLevel() >= 1) { + query.trace(getName() + " dispatch response: " + result, false, 1); + } + result.trace(getName()); + } + return result; + } + + /** + * Returns a cached result, or null if no result was cached for this key + * + * @param cacheKey the cache key created from the query packet + * @param queryPacketData a serialization of the query, to avoid having to recompute this, or null if not available + * @param query the query, used for tracing, lookup of result window and result creation + */ + private Result getCached(CacheKey cacheKey, QueryPacketData queryPacketData, Query query) { + if (query.getTraceLevel() >= 6) { + query.trace("Cache key hash: " + cacheKey.hashCode(), 6); + if (query.getTraceLevel() >= 8) { + query.trace("Cache key: " + HexDump.toHexString(cacheKey.getCopyOfFullKey()), 8); + } + } + + try { + Result result = cacheLookupFirstPhase(cacheKey, queryPacketData, query, query.getOffset(), query.getHits(), query.getPresentation().getSummary()); + if (result == null) return null; + + if (isLoggingFine()) { + getLogger().fine("Result retrieved from cache: " + result); + } + if (query.getTraceLevel() >= 1) { + query.trace(getName() + " cached response: " + result, false, 1); + } + result.trace(getName()); + return result; + } + catch (IOException e) { + Result result = new Result(query); + + if (result.hits().getErrorHit() == null) { + result.hits().setError(ErrorMessage.createBackendCommunicationError( + "Fast Search (" + getName() + ") failed: " + e.getMessage())); + } + if (query.getTraceLevel() >= 1) { + query.trace(getName() + " error response: " + result, false, 1); + } + return result; + } + } + + private List<Result> partitionHits(Result result, String summaryClass) { + List<Result> parts = new ArrayList<>(); + TinyIdentitySet<Query> queryMap = new TinyIdentitySet<>(4); + + for (Iterator<Hit> itr = hitIterator(result); itr.hasNext(); ) { + Hit hit = itr.next(); + if (hit instanceof FastHit) { + FastHit fastHit = (FastHit) hit; + if (!fastHit.isFilled(summaryClass)) { + Query q = fastHit.getQuery(); + if (q == null) { + q = result.hits().getQuery(); // fallback for untagged hits + } + int idx = queryMap.indexOf(q); + if (idx < 0) { + idx = queryMap.size(); + Result r = new Result(q); + parts.add(r); + queryMap.add(q); + } + parts.get(idx).hits().add(fastHit); + } + } + } + return parts; + } + + @Override + public void fill(Result result, String summaryClass, Execution execution) { + if (result.isFilled(summaryClass)) return; // TODO: Checked in the superclass - remove + + List<Result> parts= partitionHits(result, summaryClass); + if (parts.size() > 0) { // anything to fill at all? + for (Result r : parts) { + doPartialFill(r, summaryClass); + mergeErrorsInto(result, r); + } + result.hits().setSorted(false); + result.analyzeHits(); + } + } + + private void mergeErrorsInto(Result destination, Result source) { + ErrorHit eh = source.hits().getErrorHit(); + if (eh != null) { + for (ErrorMessage error : eh.errors()) + destination.hits().addError(error); + } + } + + static void traceQuery(String sourceName, String type, Query query, int offset, int hits, int level, Optional<String> quotedSummaryClass) { + if ((query.getTraceLevel()<level) || query.properties().getBoolean(TRACE_DISABLE)) return; + + StringBuilder s = new StringBuilder(); + s.append(sourceName).append(" " + type + " to dispatch: ") + .append("query=[") + .append(query.getModel().getQueryTree().getRoot().toString()) + .append("]"); + + s.append(" timeout=").append(query.getTimeout()).append("ms"); + + s.append(" offset=") + .append(offset) + .append(" hits=") + .append(hits); + + if (query.getRanking().hasRankProfile()) { + s.append(" rankprofile[") + .append(query.getRanking().getProfile()) + .append("]"); + } + + if (query.getRanking().getFreshness() != null) { + s.append(" freshness=") + .append(query.getRanking().getFreshness().getRefTime()); + } + + if (query.getRanking().getSorting() != null) { + s.append(" sortspec=") + .append(query.getRanking().getSorting().fieldOrders().toString()); + } + + if (query.getRanking().getLocation() != null) { + s.append(" location=") + .append(query.getRanking().getLocation().toString()); + } + + List<Grouping> grouping = GroupingExecutor.getGroupingList(query); + s.append(" grouping=").append(grouping.size()).append(" : "); + for(Grouping g : grouping) { + s.append(g.toString()); + } + + if ( ! query.getRanking().getProperties().isEmpty()) { + s.append(" rankproperties=") + .append(query.getRanking().getProperties().toString()); + } + + if ( ! query.getRanking().getFeatures().isEmpty()) { + s.append(" rankfeatures=") + .append(query.getRanking().getFeatures().toString()); + } + + if (query.getModel().getRestrict() != null) { + s.append(" restrict=").append(query.getModel().getRestrict().toString()); + } + + if (quotedSummaryClass.isPresent()) { + s.append(" summary=").append(quotedSummaryClass.get()); + } + + query.trace(s.toString(), false, level); + if (query.isTraceable(level + 1)) { + query.trace("Current state of query tree: " + + new TextualQueryRepresentation(query.getModel().getQueryTree().getRoot()), + false, level+1); + } + if (query.isTraceable(level + 2)) { + query.trace("YQL+ representation: " + query.yqlRepresentation(), level+2); + } + } + + protected void addMetaInfo(Query query, QueryPacketData queryPacketData, QueryResultPacket resultPacket, Result result, boolean fromCache) { + result.setTotalHitCount(resultPacket.getTotalDocumentCount()); + + // Grouping + if (resultPacket.getGroupData() != null) { + byte[] data = resultPacket.getGroupData(); + ArrayList<Grouping> list = new ArrayList<>(); + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(ByteBuffer.wrap(data))); + int cnt = buf.getInt(null); + for (int i = 0; i < cnt; i++) { + Grouping g = new Grouping(); + g.deserialize(buf); + list.add(g); + } + GroupingListHit hit = new GroupingListHit(list, getDocsumDefinitionSet(query)); + hit.setQuery(result.getQuery()); + hit.setSource(getName()); + hit.setSourceNumber(sourceNumber); + hit.setQueryPacketData(queryPacketData); + result.hits().add(hit); + } + + if (resultPacket.getCoverageFeature()) { + result.setCoverage(new Coverage(resultPacket.getCoverageDocs(), resultPacket.getActiveDocs())); + } + } + + private boolean fillHit(FastHit hit, DocsumPacket packet, String summaryClass) { + if (packet != null) { + byte[] docsumdata = packet.getData(); + if (docsumdata.length > 0) { + decodeSummary(summaryClass, hit, docsumdata); + return true; + } + } + return false; + } + + /** + * Fills the hits. + * + * @return the number of hits that we did not return data for, i.e + * when things are working normally we return 0. + */ + protected int fillHits(Result result, int packetIndex, Packet[] packets, String summaryClass) throws IOException { + int skippedHits=0; + for (Iterator<Hit> i = hitIterator(result); i.hasNext();) { + Hit hit = i.next(); + + if (hit instanceof FastHit && !hit.isFilled(summaryClass)) { + FastHit fastHit = (FastHit) hit; + + ensureInstanceOf(DocsumPacket.class, packets[packetIndex]); + DocsumPacket docsum = (DocsumPacket) packets[packetIndex]; + + packetIndex++; + if ( ! fillHit(fastHit, docsum, summaryClass)) + skippedHits++; + } + } + result.hits().setSorted(false); + return skippedHits; + } + + /** + * Throws an IOException if the packet is not of the expected type + */ + protected final void ensureInstanceOf(Class<? extends BasicPacket> type, BasicPacket packet) throws IOException { + if ((type.isAssignableFrom(packet.getClass()))) return; + + if (packet instanceof ErrorPacket) { + ErrorPacket errorPacket=(ErrorPacket)packet; + if (errorPacket.getErrorCode() == 8) + throw new TimeoutException("Query timed out in " + getName()); + else + throw new IOException("Received error from backend in " + getName() + ": " + packet); + } else { + throw new IOException("Received " + packet + " when expecting " + type); + } + } + + private boolean addCachedHits(Result result, + PacketWrapper packetWrapper, + String summaryClass, + List<DocumentInfo> documents) { + boolean filledAllOfEm = true; + Query myQuery = result.getQuery(); + + for (DocumentInfo document : documents) { + FastHit hit = new FastHit(); + hit.setQuery(myQuery); + + hit.setUseRowInIndexUri(useRowInIndexUri(result)); + hit.setFillable(); + hit.setCached(true); + + extractDocumentInfo(hit, document); + + DocsumPacket docsum = (DocsumPacket) packetWrapper.getPacket(document.getGlobalId(), document.getPartId(), summaryClass); + + if (docsum != null) { + byte[] docsumdata = docsum.getData(); + + if (docsumdata.length > 0) { + decodeSummary(summaryClass, hit, docsumdata); + } else { + filledAllOfEm = false; + } + } else { + filledAllOfEm = false; + } + + result.hits().add(hit); + + } + + return filledAllOfEm; + } + + private boolean useRowInIndexUri(Result result) { + return ! ((result.getQuery().properties().getString(grouping) != null) || result.getQuery().properties().getBoolean(combinerows)); + } + + private void extractDocumentInfo(FastHit hit, DocumentInfo document) { + hit.setSourceNumber(sourceNumber); + hit.setSource(getName()); + + Number rank = document.getMetric(); + + hit.setRelevance(new Relevance(rank.doubleValue())); + + hit.setDistributionKey(document.getDistributionKey()); + hit.setGlobalId(document.getGlobalId()); + hit.setPartId(document.getPartId(), rowBits); + } + + protected PacketWrapper cacheLookupTwoPhase(CacheKey cacheKey, Result result, String summaryClass) { + Query query = result.getQuery(); + PacketWrapper packetWrapper = cacheControl.lookup(cacheKey, query); + + if (packetWrapper == null) { + return null; + } + if (packetWrapper.getNumPackets() != 0) { + for (Iterator<Hit> i = hitIterator(result); i.hasNext();) { + Hit hit = i.next(); + + if (hit instanceof FastHit) { + FastHit fastHit = (FastHit) hit; + DocsumPacketKey key = new DocsumPacketKey(fastHit.getGlobalId(), fastHit.getPartId(), summaryClass); + + if (fillHit(fastHit, + (DocsumPacket) packetWrapper.getPacket(key), + summaryClass)) { + fastHit.setCached(true); + } + + } + } + result.hits().setSorted(false); + result.analyzeHits(); + } + + return packetWrapper; + } + + protected DocsumDefinitionSet getDocsumDefinitionSet(Query query) { + DocumentDatabase db = getDocumentDatabase(query); + return db.getDocsumDefinitionSet(); + } + + private void decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata) { + DocumentDatabase db = getDocumentDatabase(hit.getQuery()); + hit.setField(Hit.SDDOCNAME_FIELD, db.getName()); + decodeSummary(summaryClass, hit, docsumdata, db.getDocsumDefinitionSet()); + } + + private void decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata, DocsumDefinitionSet docsumSet) { + docsumSet.lazyDecode(summaryClass, docsumdata, hit); + hit.setFilled(summaryClass); + } + + /** + * Creates unfilled hits from a List of DocumentInfo instances. Do note + * cacheKey should be available if a cache is active, even if the hit is not + * created from a cache in the current call path. + * + * @param queryPacketData binary data from first phase of search, or null + * @param cacheKey the key this hit should match in the packet cache, or null + */ + protected boolean addUnfilledHits(Result result, List<DocumentInfo> documents, boolean fromCache, QueryPacketData queryPacketData, CacheKey cacheKey) { + boolean allHitsOK = true; + Query myQuery = result.getQuery(); + + for (DocumentInfo document : documents) { + + try { + FastHit hit = new FastHit(); + hit.setQuery(myQuery); + if (queryPacketData != null) + hit.setQueryPacketData(queryPacketData); + hit.setCacheKey(cacheKey); + + hit.setUseRowInIndexUri(useRowInIndexUri(result)); + hit.setFillable(); + hit.setCached(fromCache); + + extractDocumentInfo(hit, document); + + result.hits().add(hit); + } catch (ConfigurationException e) { + allHitsOK = false; + getLogger().log(LogLevel.WARNING, "Skipping hit", e); + } catch (Exception e) { + allHitsOK = false; + getLogger().log(LogLevel.ERROR, "Skipping malformed hit", e); + } + } + return allHitsOK; + } + + @SuppressWarnings("rawtypes") + public static VespaBackEndSearcher getSearcher(String s) { + try { + Class c = Class.forName(s); + if (VespaBackEndSearcher.class.isAssignableFrom(c)) { + Constructor[] constructors = c.getConstructors(); + for (Constructor constructor : constructors) { + Class[] parameters = constructor.getParameterTypes(); + if (parameters.length == 0) { + return (VespaBackEndSearcher) constructor.newInstance(); + } + } + throw new RuntimeException("Failed initializing " + s); + + } else { + throw new RuntimeException(s + " is not com.yahoo.prelude.fastsearch.VespaBackEndSearcher"); + } + } catch (Exception e) { + throw new RuntimeException("Failure loading class " + s + ", exception :" + e); + } + } + + protected boolean isLoggingFine() { + return getLogger().isLoggable(Level.FINE); + } + public boolean isLocalDispatching() { + return localDispatching; + } + public void setLocalDispatching(boolean localDispatching) { + this.localDispatching = localDispatching; + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java new file mode 100644 index 00000000000..0ccc8b03e3b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Class representing a string field in the result set + * + */ +package com.yahoo.prelude.fastsearch; + + +import java.nio.ByteBuffer; + +import com.yahoo.io.SlowInflate; +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.text.Utf8; +import com.yahoo.data.access.Inspector; + + +/** + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class XMLField extends DocsumField implements VariableLengthField { + public XMLField(String name) { + super(name); + } + + private Object convert(String value) { + return new XMLString(value); + } + + @Override + public Object decode(ByteBuffer b) { + long dataLen = 0; + long len = ((long) b.getInt()) & 0xffffffffL; + boolean compressed; + + // if MSB is set this is a compressed field. set the compressed + // flag accordingly and decompress the data + compressed = ((len & 0x80000000) != 0); + if (compressed) { + len &= 0x7fffffff; + dataLen = b.getInt(); + len -= 4; + } + + byte[] tmp = new byte[(int) len]; + + b.get(tmp); + + if (compressed) { + SlowInflate inf = new SlowInflate(); + + tmp = inf.unpack(tmp, (int) dataLen); + } + return convert(Utf8.toString(tmp)); + } + + @Override + public Object decode(ByteBuffer b, FastHit hit) { + Object field = decode(b); + hit.setField(name, field); + return field; + } + + @Override + public String toString() { + return "field " + getName() + " type XMLString"; + } + + @Override + public int getLength(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int len = b.getInt() & 0x7fffffff; + b.position(offset + len + (Integer.SIZE >> 3)); + return len + (Integer.SIZE >> 3); + } + + @Override + public boolean isCompressed(ByteBuffer b) { + int offset = b.position(); + // MSB = compression flag, re decode + int compressed = b.getInt() & 0x80000000; + b.position(offset); + return compressed != 0; + } + + @Override + public int sizeOfLength() { + return Integer.SIZE >> 3; + } + + public Object convert(Inspector value) { + return convert(value.asString("")); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/package-info.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/package-info.java new file mode 100644 index 00000000000..b34b74ccae3 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.prelude.fastsearch; + +import com.yahoo.osgi.annotation.ExportPackage; |