From 700a5a68f095e999f85c8ca3fd34645612310019 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 5 May 2022 07:47:45 +0200 Subject: Revert "Merge pull request #22439 from vespa-engine/revert-22429-bratseth/schema-info-5-take-2" This reverts commit 3726fd997c0cbacd37e64ff493f8775fc3fcdd9f, reversing changes made to d9ea4c99b13fa83a75bd8612d5f6c4d50510e3e9. --- .../yahoo/prelude/fastsearch/DocsumDefinition.java | 92 +++++----------------- .../prelude/fastsearch/DocsumDefinitionSet.java | 38 ++++----- .../com/yahoo/prelude/fastsearch/DocsumField.java | 5 +- .../yahoo/prelude/fastsearch/DocumentDatabase.java | 44 +++-------- .../java/com/yahoo/prelude/fastsearch/FastHit.java | 10 +-- .../com/yahoo/prelude/fastsearch/FastSearcher.java | 13 ++- .../prelude/fastsearch/VespaBackEndSearcher.java | 16 ++-- 7 files changed, 69 insertions(+), 149 deletions(-) (limited to 'container-search/src/main/java/com/yahoo/prelude/fastsearch') diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 8a7d14d5f49..33b912da42f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -1,15 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.yahoo.data.access.Inspector; +import com.yahoo.search.schema.DocumentSummary; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.stream.Collectors; /** * A docsum definition which knows how to decode a certain class of document @@ -21,55 +17,26 @@ import java.util.Set; public class DocsumDefinition { private final String name; - private final ImmutableList fields; + private final Map fields; /** True if this contains dynamic fields */ private final boolean dynamic; - // Mapping between field names and their index in this.fields - private final ImmutableMap fieldNameToIndex; - - public DocsumDefinition(String name, List fields) { - this.name = name; - this.dynamic = false; - this.fields = ImmutableList.copyOf(fields); - ImmutableMap.Builder fieldNameToIndexBuilder = new ImmutableMap.Builder<>(); - int i = 0; - for (DocsumField field : fields) - fieldNameToIndexBuilder.put(field.name, i++); - this.fieldNameToIndex = fieldNameToIndexBuilder.build(); + public DocsumDefinition(DocumentSummary documentSummary) { + this.name = documentSummary.name(); + this.dynamic = documentSummary.isDynamic(); + this.fields = documentSummary.fields() + .stream() + .map(field -> DocsumField.create(field.name(), field.type().asString())) + .collect(Collectors.toUnmodifiableMap(field -> field.getName(), + field -> field)); } - DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config) { - this.name = config.name(); - - List fieldsBuilder = new ArrayList<>(); - Map fieldNameToIndexBuilder = new HashMap<>(); - boolean dynamic = false; - for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { - // no, don't switch the order of the two next lines :) - fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); - fieldsBuilder.add(DocsumField.create(field.name(), field.type())); - if (field.dynamic()) - dynamic = true; - } - this.dynamic = dynamic; - fields = ImmutableList.copyOf(fieldsBuilder); - fieldNameToIndex = ImmutableMap.copyOf(fieldNameToIndexBuilder); - } - - /** Returns the field at this index, or null if none */ - public DocsumField getField(int fieldIndex) { - if (fieldIndex >= fields.size()) return null; - return fields.get(fieldIndex); - } + public String name() { return name; } + public Map fields() { return fields; } - /** Returns the field with this name, or null if none */ - public DocsumField getField(String fieldName) { - Integer index = fieldNameToIndex.get(fieldName); - if (index == null) return null; - return getField(index); - } + /** Returns whether this summary contains one or more dynamic fields */ + public boolean isDynamic() { return dynamic; } /** * Returns the given slime value as the type specified in this, or null if the type is not known. @@ -78,35 +45,14 @@ public class DocsumDefinition { * another has not. */ public Object convert(String fieldName, Inspector value) { - DocsumField fieldType = getField(fieldName); - if (fieldType == null || ! value.valid()) return null; - return fieldType.convert(value); - } - - public Set fieldNames() { - return fieldNameToIndex.keySet(); + DocsumField field = fields.get(fieldName); + if (field == null || ! value.valid()) return null; + return field.convert(value); } @Override public String toString() { - return "docsum definition '" + getName() + "'"; - } - - public String getName() { - return name; - } - - public int getFieldCount() { - return fields.size(); - } - - public List getFields() { - return fields; - } - - /** Returns whether this summary contains one or more dynamic fields */ - public boolean isDynamic() { - return dynamic; + return "docsum definition '" + name() + "'"; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 1e0cfa3be9e..51da9965fea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.google.common.collect.ImmutableMap; +import com.yahoo.search.schema.DocumentSummary; +import com.yahoo.search.schema.Schema; import com.yahoo.slime.BinaryFormat; import com.yahoo.data.access.Inspector; import com.yahoo.slime.Slime; @@ -10,10 +11,8 @@ import com.yahoo.prelude.ConfigurationException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; import java.util.Collection; import java.util.Map; -import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.data.access.Type.OBJECT; @@ -27,16 +26,18 @@ import static com.yahoo.data.access.Type.OBJECT; public final class DocsumDefinitionSet { public static final int SLIME_MAGIC_ID = 0x55555555; - private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); private final Map definitionsByName; - public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { - this(toDocsums(config)); + public DocsumDefinitionSet(Schema schema) { + this(schema.documentSummaries().values()); } - public DocsumDefinitionSet(Collection docsumDefinitions) { - this.definitionsByName = ImmutableMap.copyOf(docsumDefinitions.stream().collect(Collectors.toMap(DocsumDefinition::getName, p -> p))); + public DocsumDefinitionSet(Collection docsumDefinitions) { + this.definitionsByName = docsumDefinitions.stream() + .map(summary -> new DocsumDefinition(summary)) + .collect(Collectors.toUnmodifiableMap(summary -> summary.name(), + summary -> summary)); } /** @@ -45,20 +46,22 @@ public final class DocsumDefinitionSet { * @throws ConfigurationException if the requested summary class is not found and there is none called "default" */ public DocsumDefinition getDocsum(String summaryClass) { + if (summaryClass == null) + summaryClass = "default"; DocsumDefinition ds = definitionsByName.get(summaryClass); - if (ds == null) { + if (ds == null) ds = definitionsByName.get("default"); - } - if (ds == null) { + if (ds == null) throw new ConfigurationException("Fetched hit with summary class " + summaryClass + ", but this summary class is not in current summary config (" + this + ")" + " (that is, you asked for something unknown, and no default was found)"); - } return ds; } /** Do we have a summary definition with the given name */ public boolean hasDocsum(String summaryClass) { + if (summaryClass == null) + summaryClass = "default"; return definitionsByName.containsKey(summaryClass); } @@ -95,7 +98,7 @@ public final class DocsumDefinitionSet { if (sb.length() != 0) { sb.append(","); } - sb.append("[").append(e.getKey()).append(",").append(e.getValue().getName()).append("]"); + sb.append("[").append(e.getKey()).append(",").append(e.getValue().name()).append("]"); } return sb.toString(); } @@ -104,13 +107,4 @@ public final class DocsumDefinitionSet { return definitionsByName.size(); } - private static Collection toDocsums(DocumentdbInfoConfig.Documentdb config) { - Collection docsums = new ArrayList<>(); - for (int i = 0; i < config.summaryclass().size(); ++i) - docsums.add(new DocsumDefinition(config.summaryclass(i))); - if (docsums.isEmpty()) - log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); - return docsums; - } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index e412258ff15..e6201a694cb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -17,7 +17,7 @@ import java.util.logging.Logger; public abstract class DocsumField { private static final Logger log = Logger.getLogger(DocsumField.class.getName()); - private static FieldFactory fieldFactory; + private static final FieldFactory fieldFactory; private static class FieldFactory { @@ -32,8 +32,7 @@ public abstract class DocsumField { DocsumField create(String typename, String name) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { - DocsumField f = constructors.get(typename).newInstance(name); - return f; + return constructors.get(typename).newInstance(name); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index 67038e0e771..72ccf4aa1dd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -2,6 +2,8 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.tensor.TensorType; import java.util.ArrayList; @@ -17,48 +19,20 @@ import java.util.stream.Collectors; */ public class DocumentDatabase { - // TODO: What about name conflicts when different search defs have the same rank profile/docsum? - public static final String MATCH_PROPERTY = "match"; public static final String SEARCH_DOC_TYPE_KEY = "documentdb.searchdoctype"; - private final String name; + private final Schema schema; private final DocsumDefinitionSet docsumDefSet; - private final Map rankProfiles; - - public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb) { - this(documentDb.name(), new DocsumDefinitionSet(documentDb), toRankProfiles(documentDb.rankprofile())); + public DocumentDatabase(Schema schema) { + this.schema = schema; + this.docsumDefSet = new DocsumDefinitionSet(schema); } - public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection rankProfiles) { - this.name = name; - this.docsumDefSet = docsumDefinitionSet; - this.rankProfiles = Map.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::name, p -> p))); - } - - public String getName() { - return name; - } + public Schema schema() { return schema; } - public DocsumDefinitionSet getDocsumDefinitionSet() { - return docsumDefSet; - } - - /** Returns an unmodifiable map of all the rank profiles in this indexed by rank profile name */ - public Map rankProfiles() { return rankProfiles; } - - private static Collection toRankProfiles(Collection rankProfileConfigList) { - List rankProfiles = new ArrayList<>(); - for (var profileConfig : rankProfileConfigList) { - var builder = new RankProfile.Builder(profileConfig.name()); - builder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); - builder.setHasRankFeatures(profileConfig.hasRankFeatures()); - for (var inputConfig : profileConfig.input()) - builder.addInput(inputConfig.name(), TensorType.fromSpec(inputConfig.type())); - rankProfiles.add(builder.build()); - } - return rankProfiles; - } + /** Returns the document summary model in this which knows how to convert serialized data to hit fields. */ + public DocsumDefinitionSet getDocsumDefinitionSet() { return docsumDefSet; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 094367dc140..a094be943a2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -183,7 +183,7 @@ public class FastHit extends Hit { /** For internal use */ public void addSummary(DocsumDefinition docsumDef, Inspector value) { if (removedFields != null) - removedFields.removeAll(docsumDef.fieldNames()); + removedFields.removeAll(docsumDef.fields().keySet()); if ( ! (summaries instanceof ArrayList) ) summaries = new ArrayList<>(8); summaries.add(0, new SummaryData(this, docsumDef, value, 1 + summaries.size())); } @@ -363,8 +363,8 @@ public class FastHit extends Hit { @Override public String toString() { - return super.toString() + " [fasthit, globalid: " + new GlobalId(globalId).toString() + ", partId: " - + partId + ", distributionkey: " + distributionKey + "]"; + return super.toString() + " [fasthit, globalid: " + new GlobalId(globalId).toString() + ", partId: " + + partId + ", distributionkey: " + distributionKey + "]"; } @Override @@ -562,7 +562,7 @@ public class FastHit extends Hit { void forEachFieldAsRaw(RawUtf8Consumer consumer) { data.traverse((ObjectTraverser)(name, value) -> { if (!shadowed(name) && !removed(name)) { - DocsumField fieldType = type.getField(name); + DocsumField fieldType = type.fields().get(name); if (fieldType != null) { if (fieldType.isString()) { byte[] utf8Value = value.asUtf8(); @@ -592,7 +592,7 @@ public class FastHit extends Hit { private boolean shadowed(String name) { if (hit.hasField(name)) return true; for (int i = 0; i < hit.summaries.size() - index; i++) { - if (hit.summaries.get(i).type.fieldNames().contains(name)) + if (hit.summaries.get(i).type.fields().containsKey(name)) return true; } return false; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 27a45753bb5..33ad8d8c9a8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -6,6 +6,7 @@ import com.yahoo.prelude.Pong; import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.SearchInvoker; @@ -15,6 +16,7 @@ import com.yahoo.search.query.Ranking; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import java.io.IOException; @@ -51,10 +53,13 @@ public class FastSearcher extends VespaBackEndSearcher { * @param clusterParams the cluster number, and other cluster backend parameters * @param documentdbInfoConfig document database parameters */ - public FastSearcher(String serverId, Dispatcher dispatcher, - SummaryParameters docSumParams, ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig) { - init(serverId, docSumParams, clusterParams, documentdbInfoConfig); + public FastSearcher(String serverId, + Dispatcher dispatcher, + SummaryParameters docSumParams, + ClusterParams clusterParams, + DocumentdbInfoConfig documentdbInfoConfig, + SchemaInfo schemaInfo) { + init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo); this.dispatcher = dispatcher; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 3847e80d3c7..21037be1a8b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -14,10 +14,12 @@ import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.PingableSearcher; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.searchlib.aggregation.Grouping; @@ -107,7 +109,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (hasLocation(query.getModel().getQueryTree())) return true; // Needed to generate ranking features? - RankProfile rankProfile = documentDb.rankProfiles().get(query.getRanking().getProfile()); + RankProfile rankProfile = documentDb.schema().rankProfiles().get(query.getRanking().getProfile()); if (rankProfile == null) return true; // stay safe if (rankProfile.hasSummaryFeatures()) return true; if (query.getRanking().getListFeatures()) return true; @@ -133,12 +135,12 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { private void resolveDocumentDatabase(Query query) { DocumentDatabase docDb = getDocumentDatabase(query); if (docDb != null) { - query.getModel().setDocumentDb(docDb.getName()); + query.getModel().setDocumentDb(docDb.schema().name()); } } public final void init(String serverId, SummaryParameters docSumParams, ClusterParams clusterParams, - DocumentdbInfoConfig documentdbInfoConfig) { + DocumentdbInfoConfig documentdbInfoConfig, SchemaInfo schemaInfo) { this.serverId = serverId; this.name = clusterParams.searcherName; @@ -148,10 +150,9 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (documentdbInfoConfig != null) { for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { - DocumentDatabase db = new DocumentDatabase(docDb); - if (documentDbs.isEmpty()) { + DocumentDatabase db = new DocumentDatabase(schemaInfo.schemas().get(docDb.name())); + if (documentDbs.isEmpty()) defaultDocumentDb = db; - } documentDbs.put(docDb.name(), db); } } @@ -159,6 +160,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { protected void transformQuery(Query query) { } + @Override public Result search(Query query, Execution execution) { // query root should not be null here Item root = query.getModel().getQueryTree().getRoot(); @@ -391,7 +393,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { private String decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata) { DocumentDatabase db = getDocumentDatabase(hit.getQuery()); - hit.setField(Hit.SDDOCNAME_FIELD, db.getName()); + hit.setField(Hit.SDDOCNAME_FIELD, db.schema().name()); return decodeSummary(summaryClass, hit, docsumdata, db.getDocsumDefinitionSet()); } -- cgit v1.2.3