diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-04-13 22:26:21 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-04-13 22:26:21 +0200 |
commit | 43e081bc5f9ad70140873efba23bb4b4549937dc (patch) | |
tree | 79fe10099e460aabb7f929d9339b930632541dc5 | |
parent | 43f61854cb785763ae19564840557a7d9e758dce (diff) |
Revert "Revert "Revert "dispatch.summaries by default when possible"""
28 files changed, 191 insertions, 268 deletions
diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index b8242ff5101..0709baa9be2 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.objects.BufferSerializer; import java.nio.ByteBuffer; import java.util.List; + /** * An "extended query" packet. This is the query packets used today, * they allow more flexible sets of parameters to be shipped with queries. @@ -25,13 +26,12 @@ import java.util.List; */ public class QueryPacket extends Packet { - private final Query query; - + final private Query query; private QueryPacketData queryPacketData; - private int sessionOffset = 0; // Start of sessionKey ignore section for cache key - private int sessionSize = 0; // Length of sessionKey ignore section for cache key - private int ignoreableOffset = 0; // Start of (hits/offset/timestamp) ignore section for cache key - private int ignoreableSize = 0; // Length of (hits/offset/timestamp) ignore section for cache key + int sessionOffset = 0; // Start of sessionKey ignore section for cache key + int sessionSize = 0; // Length of sessionKey ignore section for cache key + int ignoreableOffset = 0; // Start of (hits/offset/timestamp) ignore section for cache key + int ignoreableSize = 0; // Length of (hits/offset/timestamp) ignore section for cache key private QueryPacket(Query query) { this.query = query; @@ -80,7 +80,6 @@ public class QueryPacket extends Packet { private int getSessionKeySkipLength() { return (sessionSize > 0) ? sessionSize + 4 : 0; } - /** * Returns an opaque cache key for the query represented by this * (pre-serialized) packet. @@ -192,14 +191,14 @@ public class QueryPacket extends Packet { /** * feature bits, taken from searchlib/common/transport.h - */ - private static final int QF_PARSEDQUERY = 0x00000002; - private static final int QF_RANKP = 0x00000004; - private static final int QF_SORTSPEC = 0x00000080; - private static final int QF_LOCATION = 0x00000800; - private static final int QF_PROPERTIES = 0x00100000; - private static final int QF_GROUPSPEC = 0x00400000; - private static final int QF_SESSIONID = 0x00800000; + **/ + static final int QF_PARSEDQUERY = 0x00000002; + static final int QF_RANKP = 0x00000004; + static final int QF_SORTSPEC = 0x00000080; + static final int QF_LOCATION = 0x00000800; + static final int QF_PROPERTIES = 0x00100000; + static final int QF_GROUPSPEC = 0x00400000; + static final int QF_SESSIONID = 0x00800000; private int getFeatureInt(boolean sendSessionId) { int features = QF_PARSEDQUERY | QF_RANKP; // this bitmask means "parsed query" in query packet. @@ -216,27 +215,27 @@ public class QueryPacket extends Packet { /** * query flag bits, taken from searchlib/common/transport.h - */ - private static final int QFLAG_EXTENDED_COVERAGE = 0x00000001; - private static final int QFLAG_COVERAGE_NODES = 0x00000002; - private static final int QFLAG_ESTIMATE = 0x00000080; - private static final int QFLAG_DROP_SORTDATA = 0x00004000; - private static final int QFLAG_NO_RESULTCACHE = 0x00010000; - private static final int QFLAG_DUMP_FEATURES = 0x00040000; + **/ + static final int QFLAG_EXTENDED_COVERAGE = 0x00000001; + static final int QFLAG_COVERAGE_NODES = 0x00000002; + static final int QFLAG_ESTIMATE = 0x00000080; + static final int QFLAG_DROP_SORTDATA = 0x00004000; + static final int QFLAG_NO_RESULTCACHE = 0x00010000; + static final int QFLAG_DUMP_FEATURES = 0x00040000; private int getFlagInt() { int flags = getQueryFlags(query); queryPacketData.setQueryFlags(flags); - /* + /** * QFLAG_DROP_SORTDATA * SORTDATA is a mangling of data from the attribute vectors * which were used in the search which is byte comparable in * such a way the comparing SORTDATA for two different hits * will reproduce the order in which the data were returned when - * using sortspec. For now we simply drop these. If they - * become necessar, QueryResultPacket must be - * updated to be able to read the sort data. + * using sortspec. For now we simple drop these, but if they + * should be necessary at later date, QueryResultPacket must be + * updated to be able to parse result packets correctly. */ flags |= QFLAG_DROP_SORTDATA; return flags; @@ -265,12 +264,13 @@ public class QueryPacket extends Packet { * creating a summary request. * * @return wrapper object suitable for creating a summary fetch packet - * @throws IllegalStateException if no wrapper has been generated + * @throws IllegalStateException + * if no wrapper has been generated */ public QueryPacketData getQueryPacketData() { - if (queryPacketData == null) + if (queryPacketData == null) { throw new IllegalStateException("Trying to fetch a hit tag without having encoded the packet first."); + } return queryPacketData; } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 258f0dec9ff..2709755da5d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -13,7 +13,6 @@ import com.yahoo.container.handler.VipStatus; import com.yahoo.fs4.mplex.Backend; import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.net.HostName; -import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.IndexFacts; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index d50006fb82c..4fd0f884903 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -21,33 +21,20 @@ import java.util.Map; */ public class DocsumDefinition { - private final String name; - private final ImmutableList<DocsumField> fields; + private String name; + private final List<DocsumField> fields; /** True if this contains dynamic fields */ - private final boolean dynamic; + private boolean dynamic = false; // Mapping between field names and their index in this.fields - private final ImmutableMap<String, Integer> fieldNameToIndex; - - public DocsumDefinition(String name, List<DocsumField> fields) { - this.name = name; - this.dynamic = false; - this.fields = ImmutableList.copyOf(fields); - ImmutableMap.Builder<String, Integer> fieldNameToIndexBuilder = new ImmutableMap.Builder<>(); - int i = 0; - for (DocsumField field : fields) - fieldNameToIndexBuilder.put(field.name, i++); - this.fieldNameToIndex = fieldNameToIndexBuilder.build(); - } + private final Map<String,Integer> fieldNameToIndex; - // TODO: Remove LegacyEmulationConfig (the config, not just the usage) on Vespa 7 DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config, LegacyEmulationConfig emulConfig) { this.name = config.name(); - List<DocsumField> fieldsBuilder = new ArrayList<>(); - Map<String, Integer> fieldNameToIndexBuilder = new HashMap<>(); - boolean dynamic = false; + Map<String,Integer> fieldNameToIndexBuilder = new HashMap<>(); + for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { // no, don't switch the order of the two next lines :) fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); @@ -55,7 +42,6 @@ public class DocsumDefinition { if (field.dynamic()) dynamic = true; } - this.dynamic = dynamic; fields = ImmutableList.copyOf(fieldsBuilder); fieldNameToIndex = ImmutableMap.copyOf(fieldNameToIndexBuilder); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 8bfb1035f1c..8d882adeb02 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.google.common.collect.ImmutableMap; import com.yahoo.slime.BinaryFormat; import com.yahoo.data.access.Inspector; import com.yahoo.slime.Slime; @@ -11,12 +10,9 @@ import com.yahoo.container.search.LegacyEmulationConfig; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.data.access.Type.OBJECT; @@ -31,24 +27,17 @@ public final class DocsumDefinitionSet { public static final int SLIME_MAGIC_ID = 0x55555555; private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); - private final Map<String, DocsumDefinition> definitionsByName; + private final HashMap<String, DocsumDefinition> definitionsByName = new HashMap<>(); private final LegacyEmulationConfig emulationConfig; public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { - this(config, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); + this.emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder()); + configure(config); } public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { - this(toDocsums(config, emulConfig), emulConfig); - } - - public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions) { - this(docsumDefinitions, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } - - public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions, LegacyEmulationConfig emulConfig) { - this.definitionsByName = ImmutableMap.copyOf(docsumDefinitions.stream().collect(Collectors.toMap(DocsumDefinition::getName, p -> p))); this.emulationConfig = emulConfig; + configure(config); } /** @@ -117,13 +106,14 @@ public final class DocsumDefinitionSet { return definitionsByName.size(); } - private static Collection<DocsumDefinition> toDocsums(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { - Collection<DocsumDefinition> docsums = new ArrayList<>(); - for (int i = 0; i < config.summaryclass().size(); ++i) - docsums.add(new DocsumDefinition(config.summaryclass(i), emulConfig)); - if (docsums.isEmpty()) + private void configure(DocumentdbInfoConfig.Documentdb config) { + for (int i = 0; i < config.summaryclass().size(); ++i) { + DocumentdbInfoConfig.Documentdb.Summaryclass sc = config.summaryclass(i); + DocsumDefinition docSumDef = new DocsumDefinition(sc, emulationConfig); + definitionsByName.put(sc.name(), docSumDef); + } + if (definitionsByName.size() == 0) { log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); - return docsums; + } } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index d5e4eb75931..1e44a8fa64d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -73,8 +73,8 @@ public abstract class DocsumField { this.name = name; } - /* For unit test only */ - public static DocsumField create(String name, String typename) { + /* for unit test only */ + static DocsumField create(String name, String typename) { return create(name, typename, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); } @@ -114,7 +114,7 @@ public abstract class DocsumField { /** * Convert a generic value into an object of the appropriate type * for this field. - */ + **/ public abstract Object convert(Inspector value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index 0ae0983a1ae..3366f92384a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -4,13 +4,10 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; import com.yahoo.container.search.LegacyEmulationConfig; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Representation of a back-end document database. @@ -27,16 +24,12 @@ public class DocumentDatabase { private final String name; private final DocsumDefinitionSet docsumDefSet; - private final ImmutableMap<String, RankProfile> rankProfiles; + private final Map<String, RankProfile> rankProfiles; public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb, LegacyEmulationConfig emulConfig) { - this(documentDb.name(), new DocsumDefinitionSet(documentDb, emulConfig), toRankProfiles(documentDb.rankprofile())); - } - - public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection<RankProfile> rankProfiles) { - this.name = name; - this.docsumDefSet = docsumDefinitionSet; - this.rankProfiles = ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); + this.name = documentDb.name(); + this.docsumDefSet = new DocsumDefinitionSet(documentDb, emulConfig); + this.rankProfiles = ImmutableMap.copyOf(toRankProfiles(documentDb.rankprofile())); } public String getName() { @@ -50,14 +43,11 @@ public class DocumentDatabase { /** Returns an unmodifiable map of all the rank profiles in this indexed by rank profile name */ public Map<String, RankProfile> rankProfiles() { return rankProfiles; } - private static ImmutableMap<String, RankProfile> toMap(Collection<RankProfile> rankProfiles) { - return ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); - } - - private static Collection<RankProfile> toRankProfiles(Collection<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { - List<RankProfile> rankProfiles = new ArrayList<>(); - for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) - rankProfiles.add(new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); + private Map<String, RankProfile> toRankProfiles(List<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { + Map<String, RankProfile> rankProfiles = new HashMap<>(); + for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) { + rankProfiles.put(c.name(), new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); + } return rankProfiles; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index b3eaee8698a..aab8aae6025 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -74,10 +74,9 @@ public class FastHit extends Hit { setPartId(0, 0); } - @Override public String toString() { return super.toString() + " [fasthit, globalid: " + globalId + ", partId: " - + partId + ", distributionkey: " + distributionKey + "]"; + + partId + ", distributionkey: " + distributionKey + "]"; } public static String asHexString(GlobalId gid) { @@ -268,8 +267,7 @@ public class FastHit extends Hit { this.distributionKey = distributionKey; } - /** For internal use */ - public void addSummary(DocsumDefinition docsumDef, Inspector value) { + void addSummary(DocsumDefinition docsumDef, Inspector value) { reserve(docsumDef.getFieldCount()); for (DocsumField field : docsumDef.getFields()) { String fieldName = field.getName(); @@ -292,8 +290,10 @@ public class FastHit extends Hit { * easier. This is not a method to be used for efficiency, as it causes * object allocations. * - * @param fieldName the name of the field to insert undecoded UTF-8 into - * @param value an array of valid UTF-8 data + * @param fieldName + * the name of the field to insert undecoded UTF-8 into + * @param value + * an array of valid UTF-8 data */ @Beta public void setLazyStringField(String fieldName, byte[] value) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index b2231fd3387..7d822fd603b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -56,9 +56,6 @@ public class FastSearcher extends VespaBackEndSearcher { /** If this is turned on this will make search queries directly to the local search node when possible */ private final static CompoundName dispatchDirect = new CompoundName("dispatch.direct"); - /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */ - private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); - /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ private final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -258,11 +255,11 @@ public class FastSearcher extends VespaBackEndSearcher { Query query = result.getQuery(); traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass)); - if (query.properties().getBoolean(dispatchSummaries, true) && !summaryNeedsQuery(query)) { + if (wantsRPCSummaryFill(query)) { CompressionType compression = CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase()); fillSDDocName(result); - dispatcher.fill(result, summaryClass, getDocumentDatabase(query), compression); + dispatcher.fill(result, summaryClass, compression); return; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java index 6c73167b162..6be5a27b49b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java @@ -9,10 +9,9 @@ import com.yahoo.data.access.Inspector; /** - * @author Mathias Mølster Lidal + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00f8lster Lidal</a> */ public class FloatField extends DocsumField { - static final double EMPTY_VALUE = Float.NaN; public FloatField(String name) { @@ -27,12 +26,10 @@ public class FloatField extends DocsumField { } } - @Override public Object decode(ByteBuffer b) { return convert(b.getFloat()); } - @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); @@ -49,5 +46,4 @@ public class FloatField extends DocsumField { public Object convert(Inspector value) { return convert((float)value.asDouble(EMPTY_VALUE)); } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java index eef6fc73294..fe44597b7d7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java @@ -12,10 +12,9 @@ import com.yahoo.search.result.NanNumber; import com.yahoo.data.access.Inspector; /** - * @author Bjørn Borud + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> */ public class IntegerField extends DocsumField { - static final int EMPTY_VALUE = Integer.MIN_VALUE; public IntegerField(String name) { @@ -26,23 +25,20 @@ public class IntegerField extends DocsumField { if (value == EMPTY_VALUE) { return NanNumber.NaN; } else { - return value; + return Integer.valueOf(value); } } - @Override public Object decode(ByteBuffer b) { return convert(b.getInt()); } - @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); return field; } - @Override public String toString() { return "field " + getName() + " type int"; } @@ -57,5 +53,4 @@ public class IntegerField extends DocsumField { public Object convert(Inspector value) { return convert((int)value.asLong(EMPTY_VALUE)); } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java index 9d22168485c..08cd4ddff35 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java @@ -86,5 +86,4 @@ public class LongdataField extends DocsumField implements VariableLengthField { public Object convert(Inspector value) { return convert(value.asData(Value.empty().asData())); } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index e5c9f048e53..eac1579a821 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -55,6 +55,8 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { private static final CompoundName grouping=new CompoundName("grouping"); private static final CompoundName combinerows=new CompoundName("combinerows"); + /** If this is turned on this will fill summaries by dispatching directly to search nodes over RPC */ + private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); protected static final CompoundName PACKET_COMPRESSION_LIMIT = new CompoundName("packetcompressionlimit"); protected static final CompoundName PACKET_COMPRESSION_TYPE = new CompoundName("packetcompressiontype"); @@ -108,6 +110,10 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { protected abstract void doPartialFill(Result result, String summaryClass); + protected static boolean wantsRPCSummaryFill(Query query) { + return query.properties().getBoolean(dispatchSummaries); + } + /** * Returns whether we need to send the query when fetching summaries. * This is necessary if the query requests summary features or dynamic snippeting @@ -210,6 +216,12 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return new Result(query, ErrorMessage.createNullQuery(query.getHttpRequest().getUri().toString())); } + if (wantsRPCSummaryFill(query) && summaryNeedsQuery(query)) { + return new Result(query, ErrorMessage.createInvalidQueryParameter( + "When using dispatch.summaries and your summary/rankprofile require the query, " + + " you need to enable ranking.queryCache.")); + } + QueryRewrite.optimizeByRestrict(query); QueryRewrite.optimizeAndNot(query); QueryRewrite.collapseSingleComposites(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java index 6ca5feb610f..df8f6e92d57 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java @@ -4,13 +4,11 @@ package com.yahoo.prelude.hitfield; /** * Represents an element of a hit property * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public interface FieldPart { - - boolean isFinal(); - boolean isToken(); - String getContent(); - String toString(); - + public abstract boolean isFinal(); + public abstract boolean isToken(); + public abstract String getContent(); + public abstract String toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java index 79d871d8c74..770ac24dcab 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java @@ -10,10 +10,11 @@ import com.yahoo.prelude.searcher.JuniperSearcher; import com.yahoo.text.XML; /** - * Represents a tokenized string field in a Hit. The original raw content and the field - * name cannot be modified. But the tokenized version can be retrieved and set. + * Represents a Field in a Hit. The original raw content and the field + * name cannot be modified. But the tokenized version can be retrieved + * and set. * - * @author Lars Christian Jensen + * @author <a href="mailto:larschr@yahoo-inc.com">Lars Christian Jensen</a> */ public class HitField { @@ -344,7 +345,10 @@ public class HitField { return xml.toString(); } - /** Returns the content of this field, using the arguments as bolding tags, as an XML string */ + /** + * @return the content of this field, using the arguments as bolding + * tags, as an XML string + */ public String quotedContent(String boldOpenTag, String boldCloseTag, String separatorTag, diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java index 268e1b53459..fa0ca62405f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java @@ -2,12 +2,12 @@ package com.yahoo.prelude.hitfield; /** - * Represents an element of a hit property which is an immutable string element + * Represents an element of a hit property which is a possibly + * mutable string element * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class ImmutableFieldPart implements FieldPart { - private final String content; private final String initContent; // Whether this element represents a (part of) a token or a @@ -15,12 +15,10 @@ public class ImmutableFieldPart implements FieldPart { // parts should inherit this state from the object they were // split from. private boolean tokenOrDelimiter; - public ImmutableFieldPart(String initContent, boolean tokenOrDelimiter) { this(initContent, initContent, tokenOrDelimiter); } - public ImmutableFieldPart(String initContent, String content, boolean tokenOrDelimiter) { @@ -29,17 +27,9 @@ public class ImmutableFieldPart implements FieldPart { this.content = content; this.tokenOrDelimiter = tokenOrDelimiter; } - - @Override public boolean isFinal() { return true; } - @Override public boolean isToken() { return tokenOrDelimiter; } - @Override public String getContent() { return content; } - public String getInitContent() { return initContent; } - - @Override public String toString() { return content; } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index 06db012309e..bdd7cf04c3e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -19,7 +19,7 @@ import java.util.Iterator; /** * A JSON wrapper. Contains XML-style rendering of a JSON structure. * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class JSONString implements Inspectable { diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java index 17def6771ae..23ca8272851 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java @@ -4,31 +4,19 @@ package com.yahoo.prelude.hitfield; /** * Represents an element of a hit property which is markup, not content. * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class MarkupFieldPart implements FieldPart { - private String content; - public MarkupFieldPart(String content) { this.content = content; } - - @Override public boolean isFinal() { return true; } - // Markup is never part of tokens as such - @Override public boolean isToken() { return false; } - public void setContent(String content) { this.content = content; } - - @Override public String getContent() { return content; } - - @Override public String toString() { return content; } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java index a0c9b10c519..dbbbca63d43 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java @@ -6,8 +6,8 @@ package com.yahoo.prelude.hitfield; * * @author arnej27959 */ -public final class RawData { - +public final class RawData +{ private byte[] content; /** @@ -52,5 +52,4 @@ public final class RawData { } return buf.toString(); } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java index 58018305fef..d7607fb7100 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java @@ -5,10 +5,9 @@ package com.yahoo.prelude.hitfield; * Represents an element of a hit property which is a possibly * mutable string element * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class StringFieldPart implements FieldPart { - private String content; private final String initContent; // Whether this element represents a (part of) a token or a @@ -16,28 +15,17 @@ public class StringFieldPart implements FieldPart { // parts should inherit this state from the object they were // split from. private boolean tokenOrDelimiter; - public StringFieldPart(String content, boolean tokenOrDelimiter) { this.content = content; initContent = content; this.tokenOrDelimiter = tokenOrDelimiter; } - - @Override public boolean isFinal() { return false; } - - @Override public boolean isToken() { return tokenOrDelimiter; } - - @Override public String getContent() { return content; } - public void setContent(String content) { this.content = content; } public String getInitContent() { return initContent; } - - @Override public String toString() { return content; } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java index 99c5daa05b8..8bafbfd6ab5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java @@ -1,14 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.hitfield; +import com.yahoo.text.Utf8; import com.yahoo.text.XML; import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Type; +import com.yahoo.data.access.simple.Value; +import com.yahoo.data.access.slime.SlimeAdapter; +import java.nio.charset.StandardCharsets; + +import java.util.Iterator; import java.util.Map; /** * Utility class for converting accessible data into the historical "prelude" xml format. - */ + **/ public class XmlRenderer { public static StringBuilder render(StringBuilder target, Inspector value) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 53b46ee1344..1b775d2c46f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -9,7 +9,6 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; -import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.slime.ArrayTraverser; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.fastsearch.FS4ResourcePool; @@ -23,9 +22,12 @@ import com.yahoo.search.result.Hit; import com.yahoo.data.access.Inspector; import com.yahoo.slime.BinaryFormat; import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.search.DispatchConfig; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -36,11 +38,7 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * A dispatcher communicates with search nodes to perform queries and fill hits. - * - * This is currently not functionally complete: Queries can only be dispatched to a single node, - * and summaries can only be requested when they do not need the query. - * + * A dispatcher communicates with search nodes to (in the future) perform queries and (now) fill hits. * This class is multithread safe. * * @author bratseth @@ -84,7 +82,7 @@ public class Dispatcher extends AbstractComponent { public SearchCluster searchCluster() { return searchCluster; } /** Fills the given summary class by sending RPC requests to the right search nodes */ - public void fill(Result result, String summaryClass, DocumentDatabase documentDb, CompressionType compression) { + public void fill(Result result, String summaryClass, CompressionType compression) { try { ListMap<Integer, FastHit> hitsByNode = hitsByNode(result); @@ -92,7 +90,7 @@ public class Dispatcher extends AbstractComponent { for (Map.Entry<Integer, List<FastHit>> nodeHits : hitsByNode.entrySet()) { sendGetDocsumsRequest(nodeHits.getKey(), nodeHits.getValue(), summaryClass, compression, result, responseReceiver); } - responseReceiver.processResponses(result.getQuery(), summaryClass, documentDb); + responseReceiver.processResponses(result.getQuery(), summaryClass); } catch (TimeoutException e) { result.hits().addError(ErrorMessage.createTimeout("Summary data is incomplete: " + e.getMessage())); @@ -194,7 +192,7 @@ public class Dispatcher extends AbstractComponent { * Call this from the dispatcher thread to initiate and complete processing of responses. * This will block until all responses are available and processed, or to timeout. */ - public void processResponses(Query query, String summaryClass, DocumentDatabase documentDb) throws TimeoutException { + public void processResponses(Query query, String summaryClass) throws TimeoutException { try { int skippedHits = 0; while (outstandingResponses > 0) { @@ -205,12 +203,11 @@ public class Dispatcher extends AbstractComponent { Client.GetDocsumsResponseOrError response = responses.poll(timeLeftMs, TimeUnit.MILLISECONDS); if (response == null) throwTimeout(); - skippedHits += processResponse(response, summaryClass, documentDb); + skippedHits += processResponse(response); outstandingResponses--; } if (skippedHits != 0) { - result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit summary data for summary " + - summaryClass + " for " + skippedHits + " hits")); + result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit summary data for summary " + summaryClass + " for " + skippedHits + " hits")); } } catch (InterruptedException e) { @@ -218,9 +215,7 @@ public class Dispatcher extends AbstractComponent { } } - private int processResponse(Client.GetDocsumsResponseOrError responseOrError, - String summaryClass, - DocumentDatabase documentDb) { + private int processResponse(Client.GetDocsumsResponseOrError responseOrError) { if (responseOrError.error().isPresent()) { if (hasReportedError) return 0; String error = responseOrError.error().get(); @@ -231,7 +226,7 @@ public class Dispatcher extends AbstractComponent { Client.GetDocsumsResponse response = responseOrError.response().get(); CompressionType compression = CompressionType.valueOf(response.compression()); byte[] slimeBytes = compressor.decompress(response.compressedSlimeBytes(), compression, response.uncompressedSize()); - return fill(response.hitsContext(), summaryClass, documentDb, slimeBytes); + return fill(response.hitsContext(), slimeBytes); } return 0; } @@ -246,7 +241,7 @@ public class Dispatcher extends AbstractComponent { }); } - private int fill(List<FastHit> hits, String summaryClass, DocumentDatabase documentDb, byte[] slimeBytes) { + private int fill(List<FastHit> hits, byte[] slimeBytes) { com.yahoo.slime.Inspector root = BinaryFormat.decode(slimeBytes).get(); com.yahoo.slime.Inspector errors = root.field("errors"); boolean hasErrors = errors.valid() && (errors.entries() > 0); @@ -261,7 +256,7 @@ public class Dispatcher extends AbstractComponent { for (int i = 0; i < hits.size(); i++) { Inspector summary = summaries.entry(i).field("docsum"); if (summary.fieldCount() != 0) { - hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsumDefinition(summaryClass), summary); + fill(hits.get(i), summary); } else { skippedHits++; } @@ -269,6 +264,27 @@ public class Dispatcher extends AbstractComponent { return skippedHits; } + private void fill(FastHit hit, Inspector summary) { + hit.reserve(summary.fieldCount()); + summary.traverse((String name, Inspector value) -> { + hit.setField(name, nativeTypeOf(value)); + }); + } + + private Object nativeTypeOf(Inspector inspector) { + switch (inspector.type()) { + case ARRAY: return inspector; + case OBJECT: return inspector; + case BOOL: return inspector.asBool(); + case DATA: return inspector.asData(); + case DOUBLE: return inspector.asDouble(); + case LONG: return inspector.asLong(); + case STRING: return inspector.asString(); // TODO: Keep as utf8 + case EMPTY : return null; + default: throw new IllegalArgumentException("Unexpected Slime type " + inspector.type()); + } + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java index 2f5d833a5be..aa2de1a1298 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java @@ -4,17 +4,16 @@ package com.yahoo.search.grouping.request; /** * This class represents a constant {@link Long} value in a {@link GroupingExpression}. * - * @author Simon Thoresen + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> */ public class LongValue extends ConstantValue<Long> { /** * Constructs a new instance of this class. * - * @param value the immutable value to assign to this. + * @param value The immutable value to assign to this. */ public LongValue(long value) { super(value); } - } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index 026538eab54..6adbac56dbe 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -403,13 +403,16 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi //TODO Should it be deprecated ? public final Map<String,Object> fields() { return getUnmodifiableFieldMap(); } - /** Aallocate room for the given number of fields to avoid resizing. */ + /** + * Will preallocate in order to avoid resizing. + * @param minSize The minimum size to reserve + */ public void reserve(int minSize) { getFieldMap(minSize); } /** - * Returns an iterator over the fields of this + * Fields * * @return an iterator for traversing the fields of this hit */ @@ -660,7 +663,9 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi if (p == null) { return null; } else if (p instanceof HitField) { - return ((HitField)p).quotedContent(false); + HitField hf = (HitField) p; + + return hf.quotedContent(false); } else if (p instanceof StructuredData) { return p.toString(); } else if (p instanceof XMLString || p instanceof JSONString) { @@ -738,10 +743,9 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } /** - * For internal use only. + * For vespa internal use only. * Gives access to the modifiable backing set of filled summaries. * This set might be unmodifiable if the size is less than or equal to 1 - * * @return the set of filled summaries. */ protected final Set<String> getFilledInternal() { @@ -751,7 +755,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi private Map<String,Object> getFieldMap() { return getFieldMap(16); } - private Map<String,Object> getFieldMap(int minSize) { if (fields == null) { // Compensate for loadfactor and then some, rounded up.... diff --git a/container-search/src/main/java/com/yahoo/search/result/NanNumber.java b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java index 078ac04f85e..2103583dfa0 100644 --- a/container-search/src/main/java/com/yahoo/search/result/NanNumber.java +++ b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java @@ -4,14 +4,14 @@ package com.yahoo.search.result; /** * A class representing unset or undefined numeric values. * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ @SuppressWarnings("serial") public final class NanNumber extends Number { - public static final NanNumber NaN = new NanNumber(); - private NanNumber() { } + private NanNumber() { + } @Override public double doubleValue() { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 01c4a488acd..20150fc2671 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.prelude.fastsearch.test; import com.google.common.collect.ImmutableList; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.container.handler.VipStatus; import com.yahoo.container.search.Fs4Config; import com.yahoo.fs4.mplex.*; import com.yahoo.fs4.test.QueryTestCase; @@ -15,7 +14,6 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.container.protect.Error; import com.yahoo.fs4.*; import com.yahoo.prelude.fastsearch.test.fs4mock.MockBackend; -import com.yahoo.prelude.fastsearch.test.fs4mock.MockFS4ResourcePool; import com.yahoo.prelude.fastsearch.test.fs4mock.MockFSChannel; import com.yahoo.processing.execution.Execution.Trace; import com.yahoo.search.Query; @@ -113,45 +111,30 @@ public class FastSearcherTestCase { .summaryclass(new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder().name("simple").id(7)) .rankprofile(new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder() .name("simpler").hasRankFeatures(false).hasSummaryFeatures(false)))); - - List<SearchCluster.Node> nodes = new ArrayList<>(); - nodes.add(new SearchCluster.Node("host1", 5000, 0)); - nodes.add(new SearchCluster.Node("host2", 5000, 0)); - - MockFS4ResourcePool mockFs4ResourcePool = new MockFS4ResourcePool(); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), - mockFs4ResourcePool, - new MockDispatcher(nodes, mockFs4ResourcePool, 1, new VipStatus()), + new FS4ResourcePool(1), + new MockDispatcher(Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(100, 1e64), documentdbConfigWithOneDb); - { // No direct.summaries - String query = "?query=sddocname:a&summary=simple"; - Result result = doSearch(fastSearcher, new Query(query), 0, 10); - doFill(fastSearcher, result); - ErrorMessage error = result.hits().getError(); - assertNull("Since we don't route to the dispatcher we hit the mock backend, so no error", error); - } + String query = "?query=sddocname:a&dispatch.summaries"; + Result result = doSearch(fastSearcher,new Query(query), 0, 10); + ErrorMessage message = result.hits().getError(); - { // direct.summaries due to query cache - String query = "?query=sddocname:a&ranking.queryCache"; - Result result = doSearch(fastSearcher, new Query(query), 0, 10); - doFill(fastSearcher, result); - ErrorMessage error = result.hits().getError(); - assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used", - "Error response from rpc node connection to host1:0: Connection error", error.getDetailedMessage()); - } + assertNotNull("Got error", message); + assertEquals("Invalid query parameter", message.getMessage()); + assertEquals("When using dispatch.summaries and your summary/rankprofile require the query, you need to enable ranking.queryCache.", message.getDetailedMessage()); + assertEquals(Error.INVALID_QUERY_PARAMETER.code, message.getCode()); - { // direct.summaries due to no summary features - String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler"; - Result result = doSearch(fastSearcher, new Query(query), 0, 10); - doFill(fastSearcher, result); - ErrorMessage error = result.hits().getError(); - assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used", - "Error response from rpc node connection to host1:0: Connection error", error.getDetailedMessage()); - } + query = "?query=sddocname:a&dispatch.summaries&ranking.queryCache"; + result = doSearch(fastSearcher,new Query(query), 0, 10); + assertNull(result.hits().getError()); + + query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler"; + result = doSearch(fastSearcher,new Query(query), 0, 10); + assertNull(result.hits().getError()); } @Test @@ -298,7 +281,7 @@ public class FastSearcherTestCase { assertEquals(100, fastSearcher.getCacheControl().capacity()); // Default cache =100MB - Query query = new Query("?query=ignored&dispatch.summaries=false"); + Query query = new Query("?query=ignored"); query.getRanking().setQueryCache(true); Result result = doSearch(fastSearcher, query, 0, 10); @@ -338,6 +321,7 @@ public class FastSearcherTestCase { answer.flip(); answer.get(expected); + assertEquals(expected.length, actual.length); for (int i = 0; i < expected.length; ++i) { if (expected[i] == IGNORE) { actual[i] = IGNORE; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java index 37a74831e56..8d705406774 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java @@ -96,13 +96,13 @@ public class MockFSChannel extends FS4Channel { && lastQueryPacket.getLastOffset() >= 1) { result.addDocument( new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(123), - 2003, 234, 0)); + 2003, 234, 1000)); } if (lastQueryPacket.getOffset() <= 1 && lastQueryPacket.getLastOffset() >= 2) { result.addDocument( new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(456), - 1855, 234, 1)); + 1855, 234, 1001)); } packets.add(result); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java index ed421be1947..9938498fa1a 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java @@ -2,11 +2,6 @@ package com.yahoo.search.dispatch; import com.yahoo.compress.CompressionType; -import com.yahoo.log.event.Collection; -import com.yahoo.prelude.fastsearch.DocsumDefinition; -import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; -import com.yahoo.prelude.fastsearch.DocsumField; -import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -15,10 +10,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; @@ -52,8 +44,7 @@ public class FillTestCase { client.setDocsumReponse("host1", 2, "summaryClass1", map("field1", "s.1.2", "field2", 2)); client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); client.setDocsumReponse("host0", 4, "summaryClass1", map("field1", "s.0.4", "field2", 4)); - - dispatcher.fill(result, "summaryClass1", db(), CompressionType.valueOf("LZ4")); + dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); @@ -88,7 +79,7 @@ public class FillTestCase { client.setDocsumReponse("host1", 2, "summaryClass1", new HashMap<>()); client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); client.setDocsumReponse("host0", 4, "summaryClass1",new HashMap<>()); - dispatcher.fill(result, "summaryClass1", db(), CompressionType.valueOf("LZ4")); + dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); @@ -117,20 +108,11 @@ public class FillTestCase { Result result = new Result(query); result.hits().add(createHit(0, 0)); - dispatcher.fill(result, "summaryClass1", db(), CompressionType.valueOf("LZ4")); + dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); assertEquals("Malfunctioning", result.hits().getError().getDetailedMessage()); } - private DocumentDatabase db() { - List<DocsumField> fields = new ArrayList<>(); - fields.add(DocsumField.create("field1", "string")); - fields.add(DocsumField.create("field2", "int64")); - DocsumDefinitionSet docsums = new DocsumDefinitionSet(Collections.singleton(new DocsumDefinition("summaryClass1", - fields))); - return new DocumentDatabase("default", docsums, Collections.emptySet()); - } - private FastHit createHit(int sourceNodeId, int hitId) { FastHit hit = new FastHit("hit:" + hitId, 1.0); hit.setPartId(sourceNodeId, 0); diff --git a/vespajlib/src/main/java/com/yahoo/text/XML.java b/vespajlib/src/main/java/com/yahoo/text/XML.java index f4cd355b0e1..30f30b2a270 100644 --- a/vespajlib/src/main/java/com/yahoo/text/XML.java +++ b/vespajlib/src/main/java/com/yahoo/text/XML.java @@ -28,18 +28,17 @@ import org.xml.sax.SAXParseException; * @author Steinar Knutsen */ public class XML { - /** * The point of this weird class and the jumble of abstract methods is * linking the scan for characters that must be quoted into the quoting * table, and making it actual work to make them go out of sync again. */ private static abstract class LegalCharacters { - // To quote http://www.w3.org/TR/REC-xml/ : // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | // [#x10000-#x10FFFF] - final boolean isLegal(int codepoint, boolean escapeLow, int stripCodePoint, boolean isAttribute) { + final boolean isLegal(final int codepoint, final boolean escapeLow, + final int stripCodePoint, final boolean isAttribute) { if (codepoint == stripCodePoint) { return removeCodePoint(); } else if (codepoint < ' ') { @@ -76,7 +75,7 @@ public class XML { } } - private boolean quotCodePoint(boolean isAttribute) { + private boolean quotCodePoint(final boolean isAttribute) { if (isAttribute) { quoteQuot(); return false; @@ -85,7 +84,7 @@ public class XML { } } - private boolean filterCodePoint(int codepoint) { + private boolean filterCodePoint(final int codepoint) { replace(codepoint); return false; } @@ -105,7 +104,7 @@ public class XML { return false; } - private boolean ctrlEscapeCodePoint(int codepoint) { + private boolean ctrlEscapeCodePoint(final int codepoint) { ctrlEscape(codepoint); return false; } @@ -350,7 +349,8 @@ public class XML { * </ul> * with character entities. * - * @param stripCodePoint any occurrence of this character is removed from the string + * @param stripCodePoint + * any occurrence of this character is removed from the string */ public static String xmlEscape(String string, boolean isAttribute, boolean escapeLowAscii, StringBuilder buffer, int stripCodePoint) { @@ -399,7 +399,8 @@ public class XML { /** * Returns the Document of an XML file reader * - * @throws RuntimeException if the root Document cannot be returned + * @throws RuntimeException + * if the root Document cannot be returned */ public static Document getDocument(Reader reader) { try { |