From ac1c5c234d8388ffcbd1da3a9ba492e3dcfc9dac Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 12 Apr 2018 20:47:19 +0200 Subject: Decode docsums the old fashioned way --- .../src/main/java/com/yahoo/fs4/QueryPacket.java | 58 +++++++++++----------- .../com/yahoo/prelude/cluster/ClusterSearcher.java | 1 + .../yahoo/prelude/fastsearch/DocsumDefinition.java | 26 +++++++--- .../prelude/fastsearch/DocsumDefinitionSet.java | 34 ++++++++----- .../com/yahoo/prelude/fastsearch/DocsumField.java | 6 +-- .../yahoo/prelude/fastsearch/DocumentDatabase.java | 28 +++++++---- .../java/com/yahoo/prelude/fastsearch/FastHit.java | 12 ++--- .../com/yahoo/prelude/fastsearch/FastSearcher.java | 2 +- .../com/yahoo/prelude/fastsearch/FloatField.java | 6 ++- .../com/yahoo/prelude/fastsearch/IntegerField.java | 9 +++- .../yahoo/prelude/fastsearch/LongdataField.java | 1 + .../java/com/yahoo/prelude/hitfield/FieldPart.java | 12 +++-- .../java/com/yahoo/prelude/hitfield/HitField.java | 12 ++--- .../yahoo/prelude/hitfield/ImmutableFieldPart.java | 16 ++++-- .../com/yahoo/prelude/hitfield/JSONString.java | 2 +- .../yahoo/prelude/hitfield/MarkupFieldPart.java | 14 +++++- .../java/com/yahoo/prelude/hitfield/RawData.java | 5 +- .../yahoo/prelude/hitfield/StringFieldPart.java | 14 +++++- .../com/yahoo/prelude/hitfield/XmlRenderer.java | 9 +--- .../java/com/yahoo/search/dispatch/Dispatcher.java | 46 +++++------------ .../yahoo/search/grouping/request/LongValue.java | 5 +- .../src/main/java/com/yahoo/search/result/Hit.java | 15 +++--- .../java/com/yahoo/search/result/NanNumber.java | 6 +-- 23 files changed, 194 insertions(+), 145 deletions(-) (limited to 'container-search/src/main/java/com') diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index 0709baa9be2..b8242ff5101 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.objects.BufferSerializer; import java.nio.ByteBuffer; import java.util.List; - /** * An "extended query" packet. This is the query packets used today, * they allow more flexible sets of parameters to be shipped with queries. @@ -26,12 +25,13 @@ import java.util.List; */ public class QueryPacket extends Packet { - final private Query query; + private final Query query; + private QueryPacketData queryPacketData; - int sessionOffset = 0; // Start of sessionKey ignore section for cache key - int sessionSize = 0; // Length of sessionKey ignore section for cache key - int ignoreableOffset = 0; // Start of (hits/offset/timestamp) ignore section for cache key - int ignoreableSize = 0; // Length of (hits/offset/timestamp) ignore section for cache key + private int sessionOffset = 0; // Start of sessionKey ignore section for cache key + private int sessionSize = 0; // Length of sessionKey ignore section for cache key + private int ignoreableOffset = 0; // Start of (hits/offset/timestamp) ignore section for cache key + private int ignoreableSize = 0; // Length of (hits/offset/timestamp) ignore section for cache key private QueryPacket(Query query) { this.query = query; @@ -80,6 +80,7 @@ public class QueryPacket extends Packet { private int getSessionKeySkipLength() { return (sessionSize > 0) ? sessionSize + 4 : 0; } + /** * Returns an opaque cache key for the query represented by this * (pre-serialized) packet. @@ -191,14 +192,14 @@ public class QueryPacket extends Packet { /** * feature bits, taken from searchlib/common/transport.h - **/ - static final int QF_PARSEDQUERY = 0x00000002; - static final int QF_RANKP = 0x00000004; - static final int QF_SORTSPEC = 0x00000080; - static final int QF_LOCATION = 0x00000800; - static final int QF_PROPERTIES = 0x00100000; - static final int QF_GROUPSPEC = 0x00400000; - static final int QF_SESSIONID = 0x00800000; + */ + private static final int QF_PARSEDQUERY = 0x00000002; + private static final int QF_RANKP = 0x00000004; + private static final int QF_SORTSPEC = 0x00000080; + private static final int QF_LOCATION = 0x00000800; + private static final int QF_PROPERTIES = 0x00100000; + private static final int QF_GROUPSPEC = 0x00400000; + private static final int QF_SESSIONID = 0x00800000; private int getFeatureInt(boolean sendSessionId) { int features = QF_PARSEDQUERY | QF_RANKP; // this bitmask means "parsed query" in query packet. @@ -215,27 +216,27 @@ public class QueryPacket extends Packet { /** * query flag bits, taken from searchlib/common/transport.h - **/ - static final int QFLAG_EXTENDED_COVERAGE = 0x00000001; - static final int QFLAG_COVERAGE_NODES = 0x00000002; - static final int QFLAG_ESTIMATE = 0x00000080; - static final int QFLAG_DROP_SORTDATA = 0x00004000; - static final int QFLAG_NO_RESULTCACHE = 0x00010000; - static final int QFLAG_DUMP_FEATURES = 0x00040000; + */ + private static final int QFLAG_EXTENDED_COVERAGE = 0x00000001; + private static final int QFLAG_COVERAGE_NODES = 0x00000002; + private static final int QFLAG_ESTIMATE = 0x00000080; + private static final int QFLAG_DROP_SORTDATA = 0x00004000; + private static final int QFLAG_NO_RESULTCACHE = 0x00010000; + private static final int QFLAG_DUMP_FEATURES = 0x00040000; private int getFlagInt() { int flags = getQueryFlags(query); queryPacketData.setQueryFlags(flags); - /** + /* * QFLAG_DROP_SORTDATA * SORTDATA is a mangling of data from the attribute vectors * which were used in the search which is byte comparable in * such a way the comparing SORTDATA for two different hits * will reproduce the order in which the data were returned when - * using sortspec. For now we simple drop these, but if they - * should be necessary at later date, QueryResultPacket must be - * updated to be able to parse result packets correctly. + * using sortspec. For now we simply drop these. If they + * become necessar, QueryResultPacket must be + * updated to be able to read the sort data. */ flags |= QFLAG_DROP_SORTDATA; return flags; @@ -264,13 +265,12 @@ public class QueryPacket extends Packet { * creating a summary request. * * @return wrapper object suitable for creating a summary fetch packet - * @throws IllegalStateException - * if no wrapper has been generated + * @throws IllegalStateException if no wrapper has been generated */ public QueryPacketData getQueryPacketData() { - if (queryPacketData == null) { + if (queryPacketData == null) throw new IllegalStateException("Trying to fetch a hit tag without having encoded the packet first."); - } return queryPacketData; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 2709755da5d..258f0dec9ff 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -13,6 +13,7 @@ import com.yahoo.container.handler.VipStatus; import com.yahoo.fs4.mplex.Backend; import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.net.HostName; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.IndexFacts; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 4fd0f884903..d50006fb82c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -21,20 +21,33 @@ import java.util.Map; */ public class DocsumDefinition { - private String name; - private final List fields; + private final String name; + private final ImmutableList fields; /** True if this contains dynamic fields */ - private boolean dynamic = false; + private final boolean dynamic; // Mapping between field names and their index in this.fields - private final Map fieldNameToIndex; + private final ImmutableMap fieldNameToIndex; + + public DocsumDefinition(String name, List fields) { + this.name = name; + this.dynamic = false; + this.fields = ImmutableList.copyOf(fields); + ImmutableMap.Builder fieldNameToIndexBuilder = new ImmutableMap.Builder<>(); + int i = 0; + for (DocsumField field : fields) + fieldNameToIndexBuilder.put(field.name, i++); + this.fieldNameToIndex = fieldNameToIndexBuilder.build(); + } + // TODO: Remove LegacyEmulationConfig (the config, not just the usage) on Vespa 7 DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config, LegacyEmulationConfig emulConfig) { this.name = config.name(); - List fieldsBuilder = new ArrayList<>(); - Map fieldNameToIndexBuilder = new HashMap<>(); + List fieldsBuilder = new ArrayList<>(); + Map fieldNameToIndexBuilder = new HashMap<>(); + boolean dynamic = false; for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { // no, don't switch the order of the two next lines :) fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); @@ -42,6 +55,7 @@ public class DocsumDefinition { if (field.dynamic()) dynamic = true; } + this.dynamic = dynamic; fields = ImmutableList.copyOf(fieldsBuilder); fieldNameToIndex = ImmutableMap.copyOf(fieldNameToIndexBuilder); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 8d882adeb02..8bfb1035f1c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; +import com.google.common.collect.ImmutableMap; import com.yahoo.slime.BinaryFormat; import com.yahoo.data.access.Inspector; import com.yahoo.slime.Slime; @@ -10,9 +11,12 @@ import com.yahoo.container.search.LegacyEmulationConfig; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.data.access.Type.OBJECT; @@ -27,17 +31,24 @@ public final class DocsumDefinitionSet { public static final int SLIME_MAGIC_ID = 0x55555555; private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); - private final HashMap definitionsByName = new HashMap<>(); + private final Map definitionsByName; private final LegacyEmulationConfig emulationConfig; public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { - this.emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder()); - configure(config); + this(config, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); } public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + this(toDocsums(config, emulConfig), emulConfig); + } + + public DocsumDefinitionSet(Collection docsumDefinitions) { + this(docsumDefinitions, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); + } + + public DocsumDefinitionSet(Collection docsumDefinitions, LegacyEmulationConfig emulConfig) { + this.definitionsByName = ImmutableMap.copyOf(docsumDefinitions.stream().collect(Collectors.toMap(DocsumDefinition::getName, p -> p))); this.emulationConfig = emulConfig; - configure(config); } /** @@ -106,14 +117,13 @@ public final class DocsumDefinitionSet { return definitionsByName.size(); } - private void configure(DocumentdbInfoConfig.Documentdb config) { - for (int i = 0; i < config.summaryclass().size(); ++i) { - DocumentdbInfoConfig.Documentdb.Summaryclass sc = config.summaryclass(i); - DocsumDefinition docSumDef = new DocsumDefinition(sc, emulationConfig); - definitionsByName.put(sc.name(), docSumDef); - } - if (definitionsByName.size() == 0) { + private static Collection toDocsums(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + Collection docsums = new ArrayList<>(); + for (int i = 0; i < config.summaryclass().size(); ++i) + docsums.add(new DocsumDefinition(config.summaryclass(i), emulConfig)); + if (docsums.isEmpty()) log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); - } + return docsums; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 1e44a8fa64d..d5e4eb75931 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -73,8 +73,8 @@ public abstract class DocsumField { this.name = name; } - /* for unit test only */ - static DocsumField create(String name, String typename) { + /* For unit test only */ + public static DocsumField create(String name, String typename) { return create(name, typename, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); } @@ -114,7 +114,7 @@ public abstract class DocsumField { /** * Convert a generic value into an object of the appropriate type * for this field. - **/ + */ public abstract Object convert(Inspector value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index 3366f92384a..0ae0983a1ae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -4,10 +4,13 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; import com.yahoo.container.search.LegacyEmulationConfig; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Representation of a back-end document database. @@ -24,12 +27,16 @@ public class DocumentDatabase { private final String name; private final DocsumDefinitionSet docsumDefSet; - private final Map rankProfiles; + private final ImmutableMap rankProfiles; public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb, LegacyEmulationConfig emulConfig) { - this.name = documentDb.name(); - this.docsumDefSet = new DocsumDefinitionSet(documentDb, emulConfig); - this.rankProfiles = ImmutableMap.copyOf(toRankProfiles(documentDb.rankprofile())); + this(documentDb.name(), new DocsumDefinitionSet(documentDb, emulConfig), toRankProfiles(documentDb.rankprofile())); + } + + public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection rankProfiles) { + this.name = name; + this.docsumDefSet = docsumDefinitionSet; + this.rankProfiles = ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); } public String getName() { @@ -43,11 +50,14 @@ public class DocumentDatabase { /** Returns an unmodifiable map of all the rank profiles in this indexed by rank profile name */ public Map rankProfiles() { return rankProfiles; } - private Map toRankProfiles(List rankProfileConfigList) { - Map rankProfiles = new HashMap<>(); - for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) { - rankProfiles.put(c.name(), new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); - } + private static ImmutableMap toMap(Collection rankProfiles) { + return ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); + } + + private static Collection toRankProfiles(Collection rankProfileConfigList) { + List rankProfiles = new ArrayList<>(); + for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) + rankProfiles.add(new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); return rankProfiles; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index aab8aae6025..b3eaee8698a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -74,9 +74,10 @@ public class FastHit extends Hit { setPartId(0, 0); } + @Override public String toString() { return super.toString() + " [fasthit, globalid: " + globalId + ", partId: " - + partId + ", distributionkey: " + distributionKey + "]"; + + partId + ", distributionkey: " + distributionKey + "]"; } public static String asHexString(GlobalId gid) { @@ -267,7 +268,8 @@ public class FastHit extends Hit { this.distributionKey = distributionKey; } - void addSummary(DocsumDefinition docsumDef, Inspector value) { + /** For internal use */ + public void addSummary(DocsumDefinition docsumDef, Inspector value) { reserve(docsumDef.getFieldCount()); for (DocsumField field : docsumDef.getFields()) { String fieldName = field.getName(); @@ -290,10 +292,8 @@ public class FastHit extends Hit { * easier. This is not a method to be used for efficiency, as it causes * object allocations. * - * @param fieldName - * the name of the field to insert undecoded UTF-8 into - * @param value - * an array of valid UTF-8 data + * @param fieldName the name of the field to insert undecoded UTF-8 into + * @param value an array of valid UTF-8 data */ @Beta public void setLazyStringField(String fieldName, byte[] value) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index a2fdcae4a18..b2231fd3387 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -262,7 +262,7 @@ public class FastSearcher extends VespaBackEndSearcher { CompressionType compression = CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase()); fillSDDocName(result); - dispatcher.fill(result, summaryClass, compression); + dispatcher.fill(result, summaryClass, getDocumentDatabase(query), compression); return; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java index 6be5a27b49b..6c73167b162 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java @@ -9,9 +9,10 @@ import com.yahoo.data.access.Inspector; /** - * @author Mathias M\u00f8lster Lidal + * @author Mathias Mølster Lidal */ public class FloatField extends DocsumField { + static final double EMPTY_VALUE = Float.NaN; public FloatField(String name) { @@ -26,10 +27,12 @@ public class FloatField extends DocsumField { } } + @Override public Object decode(ByteBuffer b) { return convert(b.getFloat()); } + @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); @@ -46,4 +49,5 @@ public class FloatField extends DocsumField { public Object convert(Inspector value) { return convert((float)value.asDouble(EMPTY_VALUE)); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java index fe44597b7d7..eef6fc73294 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java @@ -12,9 +12,10 @@ import com.yahoo.search.result.NanNumber; import com.yahoo.data.access.Inspector; /** - * @author Bj\u00f8rn Borud + * @author Bjørn Borud */ public class IntegerField extends DocsumField { + static final int EMPTY_VALUE = Integer.MIN_VALUE; public IntegerField(String name) { @@ -25,20 +26,23 @@ public class IntegerField extends DocsumField { if (value == EMPTY_VALUE) { return NanNumber.NaN; } else { - return Integer.valueOf(value); + return value; } } + @Override public Object decode(ByteBuffer b) { return convert(b.getInt()); } + @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); return field; } + @Override public String toString() { return "field " + getName() + " type int"; } @@ -53,4 +57,5 @@ public class IntegerField extends DocsumField { public Object convert(Inspector value) { return convert((int)value.asLong(EMPTY_VALUE)); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java index 08cd4ddff35..9d22168485c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java @@ -86,4 +86,5 @@ public class LongdataField extends DocsumField implements VariableLengthField { public Object convert(Inspector value) { return convert(value.asData(Value.empty().asData())); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java index df8f6e92d57..6ca5feb610f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java @@ -4,11 +4,13 @@ package com.yahoo.prelude.hitfield; /** * Represents an element of a hit property * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public interface FieldPart { - public abstract boolean isFinal(); - public abstract boolean isToken(); - public abstract String getContent(); - public abstract String toString(); + + boolean isFinal(); + boolean isToken(); + String getContent(); + String toString(); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java index 770ac24dcab..79d871d8c74 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java @@ -10,11 +10,10 @@ import com.yahoo.prelude.searcher.JuniperSearcher; import com.yahoo.text.XML; /** - * Represents a Field in a Hit. The original raw content and the field - * name cannot be modified. But the tokenized version can be retrieved - * and set. + * Represents a tokenized string field in a Hit. The original raw content and the field + * name cannot be modified. But the tokenized version can be retrieved and set. * - * @author Lars Christian Jensen + * @author Lars Christian Jensen */ public class HitField { @@ -345,10 +344,7 @@ public class HitField { return xml.toString(); } - /** - * @return the content of this field, using the arguments as bolding - * tags, as an XML string - */ + /** Returns the content of this field, using the arguments as bolding tags, as an XML string */ public String quotedContent(String boldOpenTag, String boldCloseTag, String separatorTag, diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java index fa0ca62405f..268e1b53459 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java @@ -2,12 +2,12 @@ package com.yahoo.prelude.hitfield; /** - * Represents an element of a hit property which is a possibly - * mutable string element + * Represents an element of a hit property which is an immutable string element * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public class ImmutableFieldPart implements FieldPart { + private final String content; private final String initContent; // Whether this element represents a (part of) a token or a @@ -15,10 +15,12 @@ public class ImmutableFieldPart implements FieldPart { // parts should inherit this state from the object they were // split from. private boolean tokenOrDelimiter; + public ImmutableFieldPart(String initContent, boolean tokenOrDelimiter) { this(initContent, initContent, tokenOrDelimiter); } + public ImmutableFieldPart(String initContent, String content, boolean tokenOrDelimiter) { @@ -27,9 +29,17 @@ public class ImmutableFieldPart implements FieldPart { this.content = content; this.tokenOrDelimiter = tokenOrDelimiter; } + + @Override public boolean isFinal() { return true; } + @Override public boolean isToken() { return tokenOrDelimiter; } + @Override public String getContent() { return content; } + public String getInitContent() { return initContent; } + + @Override public String toString() { return content; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index bdd7cf04c3e..06db012309e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -19,7 +19,7 @@ import java.util.Iterator; /** * A JSON wrapper. Contains XML-style rendering of a JSON structure. * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public class JSONString implements Inspectable { diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java index 23ca8272851..17def6771ae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java @@ -4,19 +4,31 @@ package com.yahoo.prelude.hitfield; /** * Represents an element of a hit property which is markup, not content. * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public class MarkupFieldPart implements FieldPart { + private String content; + public MarkupFieldPart(String content) { this.content = content; } + + @Override public boolean isFinal() { return true; } + // Markup is never part of tokens as such + @Override public boolean isToken() { return false; } + public void setContent(String content) { this.content = content; } + + @Override public String getContent() { return content; } + + @Override public String toString() { return content; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java index dbbbca63d43..a0c9b10c519 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java @@ -6,8 +6,8 @@ package com.yahoo.prelude.hitfield; * * @author arnej27959 */ -public final class RawData -{ +public final class RawData { + private byte[] content; /** @@ -52,4 +52,5 @@ public final class RawData } return buf.toString(); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java index d7607fb7100..58018305fef 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java @@ -5,9 +5,10 @@ package com.yahoo.prelude.hitfield; * Represents an element of a hit property which is a possibly * mutable string element * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public class StringFieldPart implements FieldPart { + private String content; private final String initContent; // Whether this element represents a (part of) a token or a @@ -15,17 +16,28 @@ public class StringFieldPart implements FieldPart { // parts should inherit this state from the object they were // split from. private boolean tokenOrDelimiter; + public StringFieldPart(String content, boolean tokenOrDelimiter) { this.content = content; initContent = content; this.tokenOrDelimiter = tokenOrDelimiter; } + + @Override public boolean isFinal() { return false; } + + @Override public boolean isToken() { return tokenOrDelimiter; } + + @Override public String getContent() { return content; } + public void setContent(String content) { this.content = content; } public String getInitContent() { return initContent; } + + @Override public String toString() { return content; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java index 8bafbfd6ab5..99c5daa05b8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java @@ -1,21 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.hitfield; -import com.yahoo.text.Utf8; import com.yahoo.text.XML; import com.yahoo.data.access.Inspector; -import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Type; -import com.yahoo.data.access.simple.Value; -import com.yahoo.data.access.slime.SlimeAdapter; -import java.nio.charset.StandardCharsets; - -import java.util.Iterator; import java.util.Map; /** * Utility class for converting accessible data into the historical "prelude" xml format. - **/ + */ public class XmlRenderer { public static StringBuilder render(StringBuilder target, Inspector value) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 9f68d327ade..53b46ee1344 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -9,6 +9,7 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; +import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.slime.ArrayTraverser; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.fastsearch.FS4ResourcePool; @@ -22,12 +23,9 @@ import com.yahoo.search.result.Hit; import com.yahoo.data.access.Inspector; import com.yahoo.slime.BinaryFormat; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.search.DispatchConfig; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -86,7 +84,7 @@ public class Dispatcher extends AbstractComponent { public SearchCluster searchCluster() { return searchCluster; } /** Fills the given summary class by sending RPC requests to the right search nodes */ - public void fill(Result result, String summaryClass, CompressionType compression) { + public void fill(Result result, String summaryClass, DocumentDatabase documentDb, CompressionType compression) { try { ListMap hitsByNode = hitsByNode(result); @@ -94,7 +92,7 @@ public class Dispatcher extends AbstractComponent { for (Map.Entry> nodeHits : hitsByNode.entrySet()) { sendGetDocsumsRequest(nodeHits.getKey(), nodeHits.getValue(), summaryClass, compression, result, responseReceiver); } - responseReceiver.processResponses(result.getQuery(), summaryClass); + responseReceiver.processResponses(result.getQuery(), summaryClass, documentDb); } catch (TimeoutException e) { result.hits().addError(ErrorMessage.createTimeout("Summary data is incomplete: " + e.getMessage())); @@ -196,7 +194,7 @@ public class Dispatcher extends AbstractComponent { * Call this from the dispatcher thread to initiate and complete processing of responses. * This will block until all responses are available and processed, or to timeout. */ - public void processResponses(Query query, String summaryClass) throws TimeoutException { + public void processResponses(Query query, String summaryClass, DocumentDatabase documentDb) throws TimeoutException { try { int skippedHits = 0; while (outstandingResponses > 0) { @@ -207,11 +205,12 @@ public class Dispatcher extends AbstractComponent { Client.GetDocsumsResponseOrError response = responses.poll(timeLeftMs, TimeUnit.MILLISECONDS); if (response == null) throwTimeout(); - skippedHits += processResponse(response); + skippedHits += processResponse(response, summaryClass, documentDb); outstandingResponses--; } if (skippedHits != 0) { - result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit summary data for summary " + summaryClass + " for " + skippedHits + " hits")); + result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit summary data for summary " + + summaryClass + " for " + skippedHits + " hits")); } } catch (InterruptedException e) { @@ -219,7 +218,9 @@ public class Dispatcher extends AbstractComponent { } } - private int processResponse(Client.GetDocsumsResponseOrError responseOrError) { + private int processResponse(Client.GetDocsumsResponseOrError responseOrError, + String summaryClass, + DocumentDatabase documentDb) { if (responseOrError.error().isPresent()) { if (hasReportedError) return 0; String error = responseOrError.error().get(); @@ -230,7 +231,7 @@ public class Dispatcher extends AbstractComponent { Client.GetDocsumsResponse response = responseOrError.response().get(); CompressionType compression = CompressionType.valueOf(response.compression()); byte[] slimeBytes = compressor.decompress(response.compressedSlimeBytes(), compression, response.uncompressedSize()); - return fill(response.hitsContext(), slimeBytes); + return fill(response.hitsContext(), summaryClass, documentDb, slimeBytes); } return 0; } @@ -245,7 +246,7 @@ public class Dispatcher extends AbstractComponent { }); } - private int fill(List hits, byte[] slimeBytes) { + private int fill(List hits, String summaryClass, DocumentDatabase documentDb, byte[] slimeBytes) { com.yahoo.slime.Inspector root = BinaryFormat.decode(slimeBytes).get(); com.yahoo.slime.Inspector errors = root.field("errors"); boolean hasErrors = errors.valid() && (errors.entries() > 0); @@ -260,7 +261,7 @@ public class Dispatcher extends AbstractComponent { for (int i = 0; i < hits.size(); i++) { Inspector summary = summaries.entry(i).field("docsum"); if (summary.fieldCount() != 0) { - fill(hits.get(i), summary); + hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsumDefinition(summaryClass), summary); } else { skippedHits++; } @@ -268,27 +269,6 @@ public class Dispatcher extends AbstractComponent { return skippedHits; } - private void fill(FastHit hit, Inspector summary) { - hit.reserve(summary.fieldCount()); - summary.traverse((String name, Inspector value) -> { - hit.setField(name, nativeTypeOf(value)); - }); - } - - private Object nativeTypeOf(Inspector inspector) { - switch (inspector.type()) { - case ARRAY: return inspector; - case OBJECT: return inspector; - case BOOL: return inspector.asBool(); - case DATA: return inspector.asData(); - case DOUBLE: return inspector.asDouble(); - case LONG: return inspector.asLong(); - case STRING: return inspector.asString(); // TODO: Keep as utf8 - case EMPTY : return null; - default: throw new IllegalArgumentException("Unexpected Slime type " + inspector.type()); - } - } - } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java index aa2de1a1298..2f5d833a5be 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java @@ -4,16 +4,17 @@ package com.yahoo.search.grouping.request; /** * This class represents a constant {@link Long} value in a {@link GroupingExpression}. * - * @author Simon Thoresen + * @author Simon Thoresen */ public class LongValue extends ConstantValue { /** * Constructs a new instance of this class. * - * @param value The immutable value to assign to this. + * @param value the immutable value to assign to this. */ public LongValue(long value) { super(value); } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index 6adbac56dbe..026538eab54 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -403,16 +403,13 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable fields() { return getUnmodifiableFieldMap(); } - /** - * Will preallocate in order to avoid resizing. - * @param minSize The minimum size to reserve - */ + /** Aallocate room for the given number of fields to avoid resizing. */ public void reserve(int minSize) { getFieldMap(minSize); } /** - * Fields + * Returns an iterator over the fields of this * * @return an iterator for traversing the fields of this hit */ @@ -663,9 +660,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable getFilledInternal() { @@ -755,6 +751,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable getFieldMap() { return getFieldMap(16); } + private Map getFieldMap(int minSize) { if (fields == null) { // Compensate for loadfactor and then some, rounded up.... diff --git a/container-search/src/main/java/com/yahoo/search/result/NanNumber.java b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java index 2103583dfa0..078ac04f85e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/NanNumber.java +++ b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java @@ -4,14 +4,14 @@ package com.yahoo.search.result; /** * A class representing unset or undefined numeric values. * - * @author Steinar Knutsen + * @author Steinar Knutsen */ @SuppressWarnings("serial") public final class NanNumber extends Number { + public static final NanNumber NaN = new NanNumber(); - private NanNumber() { - } + private NanNumber() { } @Override public double doubleValue() { -- cgit v1.2.3