diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/prelude/fastsearch')
11 files changed, 105 insertions, 60 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java index c1c818848ad..b7e16b6c082 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java @@ -12,7 +12,7 @@ import com.yahoo.processing.request.CompoundName; /** * The cache control logic for FastSearcher * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class CacheControl { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 4fd0f884903..d50006fb82c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -21,20 +21,33 @@ import java.util.Map; */ public class DocsumDefinition { - private String name; - private final List<DocsumField> fields; + private final String name; + private final ImmutableList<DocsumField> fields; /** True if this contains dynamic fields */ - private boolean dynamic = false; + private final boolean dynamic; // Mapping between field names and their index in this.fields - private final Map<String,Integer> fieldNameToIndex; + private final ImmutableMap<String, Integer> fieldNameToIndex; + + public DocsumDefinition(String name, List<DocsumField> fields) { + this.name = name; + this.dynamic = false; + this.fields = ImmutableList.copyOf(fields); + ImmutableMap.Builder<String, Integer> fieldNameToIndexBuilder = new ImmutableMap.Builder<>(); + int i = 0; + for (DocsumField field : fields) + fieldNameToIndexBuilder.put(field.name, i++); + this.fieldNameToIndex = fieldNameToIndexBuilder.build(); + } + // TODO: Remove LegacyEmulationConfig (the config, not just the usage) on Vespa 7 DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config, LegacyEmulationConfig emulConfig) { this.name = config.name(); - List<DocsumField> fieldsBuilder = new ArrayList<>(); - Map<String,Integer> fieldNameToIndexBuilder = new HashMap<>(); + List<DocsumField> fieldsBuilder = new ArrayList<>(); + Map<String, Integer> fieldNameToIndexBuilder = new HashMap<>(); + boolean dynamic = false; for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { // no, don't switch the order of the two next lines :) fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); @@ -42,6 +55,7 @@ public class DocsumDefinition { if (field.dynamic()) dynamic = true; } + this.dynamic = dynamic; fields = ImmutableList.copyOf(fieldsBuilder); fieldNameToIndex = ImmutableMap.copyOf(fieldNameToIndexBuilder); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 8d882adeb02..3ecc01a5e8e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; +import com.google.common.collect.ImmutableMap; import com.yahoo.slime.BinaryFormat; import com.yahoo.data.access.Inspector; import com.yahoo.slime.Slime; @@ -10,9 +11,12 @@ import com.yahoo.container.search.LegacyEmulationConfig; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.data.access.Type.OBJECT; @@ -27,19 +31,28 @@ public final class DocsumDefinitionSet { public static final int SLIME_MAGIC_ID = 0x55555555; private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); - private final HashMap<String, DocsumDefinition> definitionsByName = new HashMap<>(); + private final Map<String, DocsumDefinition> definitionsByName; private final LegacyEmulationConfig emulationConfig; public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { - this.emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder()); - configure(config); + this(config, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); } public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + this(toDocsums(config, emulConfig), emulConfig); + } + + public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions) { + this(docsumDefinitions, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); + } + + public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions, LegacyEmulationConfig emulConfig) { + this.definitionsByName = ImmutableMap.copyOf(docsumDefinitions.stream().collect(Collectors.toMap(DocsumDefinition::getName, p -> p))); this.emulationConfig = emulConfig; - configure(config); } + LegacyEmulationConfig legacyEmulationConfig() { return emulationConfig; } + /** * Returns a docsum definition by name, or null if not found * @@ -106,14 +119,13 @@ public final class DocsumDefinitionSet { return definitionsByName.size(); } - private void configure(DocumentdbInfoConfig.Documentdb config) { - for (int i = 0; i < config.summaryclass().size(); ++i) { - DocumentdbInfoConfig.Documentdb.Summaryclass sc = config.summaryclass(i); - DocsumDefinition docSumDef = new DocsumDefinition(sc, emulationConfig); - definitionsByName.put(sc.name(), docSumDef); - } - if (definitionsByName.size() == 0) { + private static Collection<DocsumDefinition> toDocsums(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + Collection<DocsumDefinition> docsums = new ArrayList<>(); + for (int i = 0; i < config.summaryclass().size(); ++i) + docsums.add(new DocsumDefinition(config.summaryclass(i), emulConfig)); + if (docsums.isEmpty()) log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); - } + return docsums; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 1e44a8fa64d..d5e4eb75931 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -73,8 +73,8 @@ public abstract class DocsumField { this.name = name; } - /* for unit test only */ - static DocsumField create(String name, String typename) { + /* For unit test only */ + public static DocsumField create(String name, String typename) { return create(name, typename, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); } @@ -114,7 +114,7 @@ public abstract class DocsumField { /** * Convert a generic value into an object of the appropriate type * for this field. - **/ + */ public abstract Object convert(Inspector value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index 3366f92384a..0ae0983a1ae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -4,10 +4,13 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; import com.yahoo.container.search.LegacyEmulationConfig; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Representation of a back-end document database. @@ -24,12 +27,16 @@ public class DocumentDatabase { private final String name; private final DocsumDefinitionSet docsumDefSet; - private final Map<String, RankProfile> rankProfiles; + private final ImmutableMap<String, RankProfile> rankProfiles; public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb, LegacyEmulationConfig emulConfig) { - this.name = documentDb.name(); - this.docsumDefSet = new DocsumDefinitionSet(documentDb, emulConfig); - this.rankProfiles = ImmutableMap.copyOf(toRankProfiles(documentDb.rankprofile())); + this(documentDb.name(), new DocsumDefinitionSet(documentDb, emulConfig), toRankProfiles(documentDb.rankprofile())); + } + + public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection<RankProfile> rankProfiles) { + this.name = name; + this.docsumDefSet = docsumDefinitionSet; + this.rankProfiles = ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); } public String getName() { @@ -43,11 +50,14 @@ public class DocumentDatabase { /** Returns an unmodifiable map of all the rank profiles in this indexed by rank profile name */ public Map<String, RankProfile> rankProfiles() { return rankProfiles; } - private Map<String, RankProfile> toRankProfiles(List<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { - Map<String, RankProfile> rankProfiles = new HashMap<>(); - for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) { - rankProfiles.put(c.name(), new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); - } + private static ImmutableMap<String, RankProfile> toMap(Collection<RankProfile> rankProfiles) { + return ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); + } + + private static Collection<RankProfile> toRankProfiles(Collection<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { + List<RankProfile> rankProfiles = new ArrayList<>(); + for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) + rankProfiles.add(new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); return rankProfiles; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index aab8aae6025..b3eaee8698a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -74,9 +74,10 @@ public class FastHit extends Hit { setPartId(0, 0); } + @Override public String toString() { return super.toString() + " [fasthit, globalid: " + globalId + ", partId: " - + partId + ", distributionkey: " + distributionKey + "]"; + + partId + ", distributionkey: " + distributionKey + "]"; } public static String asHexString(GlobalId gid) { @@ -267,7 +268,8 @@ public class FastHit extends Hit { this.distributionKey = distributionKey; } - void addSummary(DocsumDefinition docsumDef, Inspector value) { + /** For internal use */ + public void addSummary(DocsumDefinition docsumDef, Inspector value) { reserve(docsumDef.getFieldCount()); for (DocsumField field : docsumDef.getFields()) { String fieldName = field.getName(); @@ -290,10 +292,8 @@ public class FastHit extends Hit { * easier. This is not a method to be used for efficiency, as it causes * object allocations. * - * @param fieldName - * the name of the field to insert undecoded UTF-8 into - * @param value - * an array of valid UTF-8 data + * @param fieldName the name of the field to insert undecoded UTF-8 into + * @param value an array of valid UTF-8 data */ @Beta public void setLazyStringField(String fieldName, byte[] value) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 7d822fd603b..e5f255f18c0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.fastsearch; import java.util.Optional; import com.yahoo.compress.CompressionType; +import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.fs4.BasicPacket; import com.yahoo.fs4.ChannelTimeoutException; import com.yahoo.fs4.GetDocSumsPacket; @@ -15,7 +16,6 @@ import com.yahoo.fs4.QueryResultPacket; import com.yahoo.fs4.mplex.Backend; import com.yahoo.fs4.mplex.FS4Channel; import com.yahoo.fs4.mplex.InvalidChannelException; -import com.yahoo.net.HostName; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.prelude.querytransform.QueryRewrite; @@ -56,6 +56,9 @@ public class FastSearcher extends VespaBackEndSearcher { /** If this is turned on this will make search queries directly to the local search node when possible */ private final static CompoundName dispatchDirect = new CompoundName("dispatch.direct"); + /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */ + private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); + /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ private final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -231,6 +234,7 @@ public class FastSearcher extends VespaBackEndSearcher { /** * Only used to fill the sddocname field when using direct dispatching as that is normally done in VespaBackEndSearcher.decodeSummary + * * @param result The result */ private void fillSDDocName(Result result) { @@ -255,11 +259,15 @@ public class FastSearcher extends VespaBackEndSearcher { Query query = result.getQuery(); traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass)); - if (wantsRPCSummaryFill(query)) { + if (query.properties().getBoolean(dispatchSummaries, true) + && ! summaryNeedsQuery(query) + && ! cacheControl.useCache(query) + && ! legacyEmulationConfigIsSet(getDocumentDatabase(query))) { + CompressionType compression = CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase()); fillSDDocName(result); - dispatcher.fill(result, summaryClass, compression); + dispatcher.fill(result, summaryClass, getDocumentDatabase(query), compression); return; } @@ -349,6 +357,14 @@ public class FastSearcher extends VespaBackEndSearcher { } } + private boolean legacyEmulationConfigIsSet(DocumentDatabase db) { + LegacyEmulationConfig config = db.getDocsumDefinitionSet().legacyEmulationConfig(); + if (config.forceFillEmptyFields()) return true; + if (config.stringBackedFeatureData()) return true; + if (config.stringBackedStructuredData()) return true; + return false; + } + private static @NonNull Optional<String> quotedSummaryClass(String summaryClass) { return Optional.of(summaryClass == null ? "[null]" : quote(summaryClass)); } @@ -465,14 +481,9 @@ public class FastSearcher extends VespaBackEndSearcher { } boolean couldSend = channel.sendPacket(docsumsPacket); - if (isLoggingFine()) - getLogger().finest("Sent " + docsumsPacket + " on " + channel); if ( ! couldSend) throw new IOException("Could not successfully send GetDocSumsPacket."); receivedPackets = channel.receivePackets(result.getQuery().getTimeLeft(), docsumsPacket.getNumDocsums() + 1); - if (isLoggingFine()) - getLogger().finest("got " + receivedPackets.length + "docsumPackets"); - return convertBasicPackets(receivedPackets); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java index 6be5a27b49b..6c73167b162 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java @@ -9,9 +9,10 @@ import com.yahoo.data.access.Inspector; /** - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00f8lster Lidal</a> + * @author Mathias Mølster Lidal */ public class FloatField extends DocsumField { + static final double EMPTY_VALUE = Float.NaN; public FloatField(String name) { @@ -26,10 +27,12 @@ public class FloatField extends DocsumField { } } + @Override public Object decode(ByteBuffer b) { return convert(b.getFloat()); } + @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); @@ -46,4 +49,5 @@ public class FloatField extends DocsumField { public Object convert(Inspector value) { return convert((float)value.asDouble(EMPTY_VALUE)); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java index fe44597b7d7..eef6fc73294 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java @@ -12,9 +12,10 @@ import com.yahoo.search.result.NanNumber; import com.yahoo.data.access.Inspector; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author Bjørn Borud */ public class IntegerField extends DocsumField { + static final int EMPTY_VALUE = Integer.MIN_VALUE; public IntegerField(String name) { @@ -25,20 +26,23 @@ public class IntegerField extends DocsumField { if (value == EMPTY_VALUE) { return NanNumber.NaN; } else { - return Integer.valueOf(value); + return value; } } + @Override public Object decode(ByteBuffer b) { return convert(b.getInt()); } + @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); return field; } + @Override public String toString() { return "field " + getName() + " type int"; } @@ -53,4 +57,5 @@ public class IntegerField extends DocsumField { public Object convert(Inspector value) { return convert((int)value.asLong(EMPTY_VALUE)); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java index 08cd4ddff35..9d22168485c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java @@ -86,4 +86,5 @@ public class LongdataField extends DocsumField implements VariableLengthField { public Object convert(Inspector value) { return convert(value.asData(Value.empty().asData())); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index eac1579a821..e5c9f048e53 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -55,8 +55,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { private static final CompoundName grouping=new CompoundName("grouping"); private static final CompoundName combinerows=new CompoundName("combinerows"); - /** If this is turned on this will fill summaries by dispatching directly to search nodes over RPC */ - private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); protected static final CompoundName PACKET_COMPRESSION_LIMIT = new CompoundName("packetcompressionlimit"); protected static final CompoundName PACKET_COMPRESSION_TYPE = new CompoundName("packetcompressiontype"); @@ -110,10 +108,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { protected abstract void doPartialFill(Result result, String summaryClass); - protected static boolean wantsRPCSummaryFill(Query query) { - return query.properties().getBoolean(dispatchSummaries); - } - /** * Returns whether we need to send the query when fetching summaries. * This is necessary if the query requests summary features or dynamic snippeting @@ -216,12 +210,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return new Result(query, ErrorMessage.createNullQuery(query.getHttpRequest().getUri().toString())); } - if (wantsRPCSummaryFill(query) && summaryNeedsQuery(query)) { - return new Result(query, ErrorMessage.createInvalidQueryParameter( - "When using dispatch.summaries and your summary/rankprofile require the query, " + - " you need to enable ranking.queryCache.")); - } - QueryRewrite.optimizeByRestrict(query); QueryRewrite.optimizeAndNot(query); QueryRewrite.collapseSingleComposites(query); |