diff options
51 files changed, 579 insertions, 730 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java index 3a8268029d0..99c40c2700f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java @@ -26,9 +26,17 @@ public abstract class Derived implements Exportable { private String name; + public Derived() { + this(""); + } + + public Derived(String name) { + this.name = name; + } + public String getName() { return name; } - protected final void setName(String name) { this.name=name; } + protected final void setName(String name) { this.name = name; } /** * Derives the content of this configuration. This diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 64a679ed454..48bc972f7ef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -24,7 +24,7 @@ import java.io.Writer; import java.util.concurrent.ExecutorService; /** - * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when + * A set of all derived configuration of a schema. Use this as a facade to individual configurations when * necessary. * * @author bratseth @@ -39,6 +39,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { private RankProfileList rankProfileList; private IndexingScript indexingScript; private IndexInfo indexInfo; + private SchemaInfo schemaInfo; private VsmFields streamingFields; private VsmSummary streamingSummary; private IndexSchema indexSchema; @@ -99,6 +100,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { queryProfiles, importedModels, deployProperties, executor); indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema); + schemaInfo = new SchemaInfo(schema, rankProfileRegistry, summaries, summaryMap); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); } @@ -179,6 +181,8 @@ public class DerivedConfiguration implements AttributesConfig.Producer { return indexInfo; } + public SchemaInfo getSchemaInfo() { return schemaInfo; } + public void setIndexingScript(IndexingScript script) { this.indexingScript = script; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java index 232d80d59b4..39432fd0049 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java @@ -6,19 +6,19 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; /** * The result transformation of a named field * - * @author bratseth + * @author bratseth */ public class FieldResultTransform { - private String fieldName; + private final String fieldName; private SummaryTransform transform; - private String argument; + private final String argument; - public FieldResultTransform(String fieldName,SummaryTransform transform,String argument) { - this.fieldName=fieldName; - this.transform=transform; + public FieldResultTransform(String fieldName, SummaryTransform transform, String argument) { + this.fieldName = fieldName; + this.transform = transform; this.argument = argument; } @@ -26,18 +26,19 @@ public class FieldResultTransform { public SummaryTransform getTransform() { return transform; } - public void setTransform(SummaryTransform transform) { this.transform=transform; } + public void setTransform(SummaryTransform transform) { this.transform = transform; } /** Returns the argument of this (used as input to the backend docsum rewriter) */ public String getArgument() { return argument; } public int hashCode() { - return fieldName.hashCode() + 11*transform.hashCode() + 17* argument.hashCode(); + return fieldName.hashCode() + 11 * transform.hashCode() + 17 * argument.hashCode(); } + @Override public boolean equals(Object o) { if (! (o instanceof FieldResultTransform)) return false; - FieldResultTransform other=(FieldResultTransform)o; + FieldResultTransform other = (FieldResultTransform)o; return this.fieldName.equals(other.fieldName) && @@ -45,10 +46,12 @@ public class FieldResultTransform { this.argument.equals(other.argument); } + @Override public String toString() { - String sourceString=""; + String sourceString = ""; if ( ! argument.equals(fieldName)) - sourceString=" (argument: " + argument + ")"; + sourceString = " (argument: " + argument + ")"; return "field " + fieldName + ": " + transform + sourceString; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index bcebdf3a916..cd94b4a7f6e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -41,7 +41,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private final LargeRankExpressions largeRankExpressions; private final OnnxModels onnxModels; - public static RankProfileList empty = new RankProfileList(); + public static final RankProfileList empty = new RankProfileList(); private RankProfileList() { rankingConstants = new RankingConstants(null, Optional.empty()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java new file mode 100644 index 00000000000..3698345eb11 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java @@ -0,0 +1,129 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.search.SearchCluster; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Information about a schema. + * + * @author bratseth + */ +public final class SchemaInfo implements SchemaInfoConfig.Producer { + + private final Schema schema; + + // Info about profiles needed in memory after build. + // The rank profile registry itself is not kept around due to its size. + private final Map<String, RankProfileInfo> rankProfiles; + + private final Summaries summaries; + private final SummaryMap summaryMap; + + public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, + Summaries summaries, SummaryMap summaryMap) { + this.schema = schema; + this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); + this.summaries = summaries; + this.summaryMap = summaryMap; + } + + public String name() { return schema.getName(); } + + public Schema fullSchema() { return schema; } + + public Map<String, RankProfileInfo> rankProfiles() { return rankProfiles; } + + private Map<String, RankProfileInfo> toRankProfiles(Collection<RankProfile> rankProfiles) { + Map<String, RankProfileInfo> rankProfileInfos = new LinkedHashMap<>(); + rankProfiles.forEach(profile -> rankProfileInfos.put(profile.name(), new RankProfileInfo(profile))); + return rankProfileInfos; + } + + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); + schemaBuilder.name(schema.getName()); + addSummaryConfig(schemaBuilder); + addRankProfilesConfig(schemaBuilder); + builder.schema(schemaBuilder); + } + + private void addSummaryConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (var summary : summaries.asList()) { + var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder(); + summaryBuilder.name(summary.getName()); + for (var field : summary.fields().values()) { + var fieldsBuilder = new SchemaInfoConfig.Schema.Summaryclass.Fields.Builder(); + fieldsBuilder.name(field.getName()) + .type(field.getType().getName()) + .dynamic(isDynamic(field.getName())); + summaryBuilder.fields(fieldsBuilder); + } + schemaBuilder.summaryclass(summaryBuilder); + } + } + + /** Returns whether the given field is a dynamic summary field. */ + private boolean isDynamic(String fieldName) { + if (summaryMap == null) return false; // not know for streaming, but also not used + + var fieldTransform = summaryMap.resultTransforms().get(fieldName); + if (fieldTransform == null) return false; + // TODO: Move this into SummaryTransform and call it something else than "dynamic" + return fieldTransform.getTransform().isDynamic() || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER; + } + + private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (RankProfileInfo rankProfile : rankProfiles().values()) { + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileConfig.name(rankProfile.name()); + rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); + rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + for (var input : rankProfile.inputs().entrySet()) { + var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); + inputConfig.name(input.getKey().toString()); + inputConfig.type(input.getValue().toString()); + rankProfileConfig.input(inputConfig); + } + schemaBuilder.rankprofile(rankProfileConfig); + } + } + + /** A store of a *small* (in memory) amount of rank profile info. */ + public static final class RankProfileInfo { + + private final String name; + private final boolean hasSummaryFeatures; + private final boolean hasRankFeatures; + private final Map<Reference, TensorType> inputs; + + public RankProfileInfo(RankProfile profile) { + this.name = profile.name(); + this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); + this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); + this.inputs = profile.inputs(); + } + + public String name() { return name; } + public boolean hasSummaryFeatures() { return hasSummaryFeatures; } + public boolean hasRankFeatures() { return hasRankFeatures; } + public Map<Reference, TensorType> inputs() { return inputs; } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java index 1455fbc92e1..e259c07dbb5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java @@ -6,6 +6,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.config.search.SummaryConfig; + +import java.util.ArrayList; import java.util.List; /** @@ -16,18 +18,24 @@ import java.util.List; public class Summaries extends Derived implements SummaryConfig.Producer { private final boolean useV8GeoPositions; - private List<SummaryClass> summaries=new java.util.ArrayList<>(1); + private final List<SummaryClass> summaries; public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureFlags featureFlags) { + super(); this.useV8GeoPositions = featureFlags.useV8GeoPositions(); + // Make sure the default is first + List<SummaryClass> summaries = new ArrayList<>(); summaries.add(new SummaryClass(schema, schema.getSummary("default"), deployLogger)); for (DocumentSummary summary : schema.getSummaries().values()) { if (!summary.getName().equals("default")) summaries.add(new SummaryClass(schema, summary, deployLogger)); } + this.summaries = List.copyOf(summaries); } + public List<SummaryClass> asList() { return summaries; } + @Override protected String getDerivedName() { return "summary"; } @@ -39,4 +47,5 @@ public class Summaries extends Derived implements SummaryConfig.Producer { builder.classes(summaryClass.getSummaryClassConfig()); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index 6c233aacf30..ea8e2aac1f0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -9,9 +9,9 @@ import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; -import java.util.Iterator; + +import java.util.Collections; import java.util.Map; -import java.util.Random; import java.util.logging.Level; /** @@ -19,63 +19,66 @@ import java.util.logging.Level; * Each summary definition have at least one summary, the default * which has the same name as the search definition. * - * @author bratseth + * @author bratseth */ public class SummaryClass extends Derived { public static final String DOCUMENT_ID_FIELD = "documentid"; + private final int id; + /** True if this summary class needs to access summary information on disk */ private boolean accessingDiskSummary = false; private final boolean rawAsBase64; private final boolean omitSummaryFeatures; /** The summary fields of this indexed by name */ - private final Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); + private final Map<String, SummaryClassField> fields; private final DeployLogger deployLogger; - private final Random random = new Random(7); - /** * Creates a summary class from a search definition summary * * @param deployLogger a {@link DeployLogger} */ public SummaryClass(Schema schema, DocumentSummary summary, DeployLogger deployLogger) { + super(summary.getName()); this.deployLogger = deployLogger; this.rawAsBase64 = schema.isRawAsBase64(); this.omitSummaryFeatures = summary.omitSummaryFeatures(); - deriveName(summary); - deriveFields(schema, summary); - deriveImplicitFields(summary); + Map<String, SummaryClassField> fields = new java.util.LinkedHashMap<>(); + deriveFields(schema, summary, fields); + deriveImplicitFields(summary, fields); + this.fields = Collections.unmodifiableMap(fields); + this.id = deriveId(summary.getName(), fields); } - private void deriveName(DocumentSummary summary) { - setName(summary.getName()); - } + public int id() { return id; } /** MUST be called after all other fields are added */ - private void deriveImplicitFields(DocumentSummary summary) { + private void deriveImplicitFields(DocumentSummary summary, Map<String, SummaryClassField> fields) { if (summary.getName().equals("default")) { - addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING); + addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, fields); } } - private void deriveFields(Schema schema, DocumentSummary summary) { + private void deriveFields(Schema schema, DocumentSummary summary, Map<String, SummaryClassField> fields) { for (SummaryField summaryField : summary.getSummaryFields().values()) { if (!accessingDiskSummary && schema.isAccessingDiskSummary(summaryField)) { accessingDiskSummary = true; } - addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform()); + addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform(), fields); } } - private void addField(String name, DataType type) { - addField(name, type, null); + private void addField(String name, DataType type, Map<String, SummaryClassField> fields) { + addField(name, type, null, fields); } - private void addField(String name, DataType type, SummaryTransform transform) { + private void addField(String name, DataType type, + SummaryTransform transform, + Map<String, SummaryClassField> fields) { if (fields.containsKey(name)) { SummaryClassField sf = fields.get(name); if ( SummaryClassField.convertDataType(type, transform, rawAsBase64) != sf.getType()) { @@ -87,48 +90,23 @@ public class SummaryClass extends Derived { } } + public Map<String, SummaryClassField> fields() { return fields; } - /** Returns an iterator of the fields of this summary. Removes on this iterator removes the field from this summary */ - public Iterator<SummaryClassField> fieldIterator() { - return fields.values().iterator(); - } - - public void addField(SummaryClassField field) { - fields.put(field.getName(),field); - } - - /** Returns the writable map of fields of this summary */ // TODO: Make read only, move writers to iterator/addField - public Map<String,SummaryClassField> getFields() { return fields; } - - public SummaryClassField getField(String name) { - return fields.get(name); - } - - public int getFieldCount() { return fields.size(); } - - @Override - public int hashCode() { + private static int deriveId(String name, Map<String, SummaryClassField> fields) { + int hash = name.hashCode(); int number = 1; - int hash = getName().hashCode(); - for (Iterator i = fieldIterator(); i.hasNext(); ) { - SummaryClassField field = (SummaryClassField)i.next(); - hash += number * (field.getName().hashCode() + - 17*field.getType().getName().hashCode()); - number++; + for (var field : fields.values()) { + hash += number++ * (field.getName().hashCode() + + 17 * field.getType().getName().hashCode()); } - if (hash < 0) - hash *= -1; + hash = Math.abs(hash); + if (hash == DocsumDefinitionSet.SLIME_MAGIC_ID) + hash++; return hash; } public SummaryConfig.Classes.Builder getSummaryClassConfig() { SummaryConfig.Classes.Builder classBuilder = new SummaryConfig.Classes.Builder(); - int id = hashCode(); - if (id == DocsumDefinitionSet.SLIME_MAGIC_ID) { - deployLogger.log(Level.WARNING, "Summary class '" + getName() + "' hashes to the SLIME_MAGIC_ID '" + id + - "'. This is unlikely but I autofix it for you by adding a random number."); - id += random.nextInt(); - } classBuilder. id(id). name(getName()). @@ -142,11 +120,14 @@ public class SummaryClass extends Derived { } @Override + public int hashCode() { return id; } + + @Override protected String getDerivedName() { return "summary"; } @Override public String toString() { - return "summary class " + getName(); + return "summary class '" + getName() + "'"; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java index c65ed7dc762..9e1740b4073 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java @@ -14,13 +14,13 @@ import java.util.Map; /** * A summary map (describing search-time summary field transformations) - * derived from a search definition + * derived from a Schema. * - * @author bratseth + * @author bratseth */ public class SummaryMap extends Derived implements SummarymapConfig.Producer { - private Map<String,FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>(); + private final Map<String, FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>(); /** Creates a summary map from a search definition */ SummaryMap(Schema schema) { @@ -49,32 +49,32 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { summaryField.getTransform()==SummaryTransform.MATCHED_ELEMENTS_FILTER || summaryField.getTransform()==SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) { - resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getSingleSource())); + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getSingleSource())); } else { // Note: Currently source mapping is handled in the indexing statement, // by creating a summary field for each of the values // This works, but is suboptimal. We could consolidate to a minimal set and // use the right value from the minimal set as the third parameter here, // and add "override" commands to multiple static values - resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getName())); + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getName())); } } } /** Returns a read-only iterator of the FieldResultTransforms of this summary map */ - public Iterator resultTransformIterator() { - return Collections.unmodifiableCollection(resultTransforms.values()).iterator(); + public Map<String, FieldResultTransform> resultTransforms() { + return Collections.unmodifiableMap(resultTransforms); } protected String getDerivedName() { return "summarymap"; } /** Returns the command name of a transform */ private String getCommand(SummaryTransform transform) { - if (transform.equals(SummaryTransform.DISTANCE)) + if (transform == SummaryTransform.DISTANCE) return "absdist"; else if (transform.isDynamic()) return "dynamicteaser"; @@ -87,6 +87,7 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { * We need this because some model information is shared through configs instead of model - see usage * A dynamic transform needs the query to perform its computations. */ + // TODO/Note: "dynamic" here means something else than in SummaryTransform public static boolean isDynamicCommand(String commandName) { return (commandName.equals("dynamicteaser") || commandName.equals(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()) || diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 59b4e1a2f9b..2edc845d6b7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -21,6 +21,7 @@ import java.util.logging.Level; */ public class DocumentSummary extends FieldView { + private int id; private boolean fromDisk = false; private boolean omitSummaryFeatures = false; private Optional<String> inherited = Optional.empty(); @@ -33,6 +34,8 @@ public class DocumentSummary extends FieldView { this.owner = owner; } + public int id() { return id; } + public void setFromDisk(boolean fromDisk) { this.fromDisk = fromDisk; } /** Returns whether the user has noted explicitly that this summary accesses disk */ diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java index 04f69d041b6..73d699dda1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java @@ -10,11 +10,11 @@ import java.util.Map; /** * @author baldersheim - * @since 2010-02-19 */ public class FieldView implements Serializable { - private String name; - private Map<String, Field> fields = new LinkedHashMap<>(); + + private final String name; + private final Map<String, Field> fields = new LinkedHashMap<>(); /** * Creates a view with a name diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index d814f3e5d63..c50766a2585 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -24,7 +24,7 @@ public enum SummaryTransform { MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"); - private String name; + private final String name; SummaryTransform(String name) { this.name=name; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index 65838a2c23a..d145ce6bd24 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.derived.SchemaInfo; import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; @@ -29,7 +30,7 @@ public class ComplexAttributeFieldsValidator extends Validator { for (SearchCluster cluster : searchClusters) { if (cluster.isStreaming()) continue; - for (SearchCluster.SchemaInfo spec : cluster.schemas().values()) { + for (SchemaInfo spec : cluster.schemas().values()) { validateComplexFields(cluster.getClusterName(), spec.fullSchema()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index d6a14eeb585..162cbe7495f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -12,6 +12,7 @@ import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.derived.SchemaInfo; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.VespaModel; @@ -34,7 +35,7 @@ public class SearchDataTypeValidator extends Validator { if (cluster.isStreaming()) { continue; } - for (SearchCluster.SchemaInfo spec : cluster.schemas().values()) { + for (SchemaInfo spec : cluster.schemas().values()) { SDDocumentType docType = spec.fullSchema().getDocument(); if (docType == null) { continue; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 789b0fc6fb7..e26fb51bf98 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -8,6 +8,7 @@ import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.searchdefinition.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; @@ -122,31 +123,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> @Override public void getConfig(SchemaInfoConfig.Builder builder) { - Map<String, SearchCluster.SchemaInfo> allSchemas = new LinkedHashMap<>(); - for (SearchCluster sc : searchClusters) - allSchemas.putAll(sc.schemas()); - - for (var schemaEntry : allSchemas.entrySet()) { - var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); - schemaBuilder.name(schemaEntry.getKey()); - addRankProfilesConfig(schemaEntry.getValue(), schemaBuilder); - builder.schema(schemaBuilder); - } - } - - protected void addRankProfilesConfig(SearchCluster.SchemaInfo schema, SchemaInfoConfig.Schema.Builder schemaBuilder) { - for (SearchCluster.RankProfileInfo rankProfile : schema.rankProfiles().values()) { - var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileConfig.name(rankProfile.name()); - rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); - rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); - for (var input : rankProfile.inputs().entrySet()) { - var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); - inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); - rankProfileConfig.input(inputConfig); - } - schemaBuilder.rankprofile(rankProfileConfig); + for (SearchCluster sc : searchClusters) { + sc.getConfig(builder); } } @@ -156,7 +134,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> SearchCluster sys = findClusterWithId(searchClusters, i); QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). name(sys.getClusterName()); - for (SearchCluster.SchemaInfo spec : sys.schemas().values()) { + for (SchemaInfo spec : sys.schemas().values()) { scB.searchdef(spec.fullSchema().getName()); } scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index 4a0eb5d951a..1ea84a33d2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -7,6 +7,7 @@ import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.cluster.QrMonitorConfig; +import com.yahoo.searchdefinition.derived.SchemaInfo; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.search.config.ClusterConfig; @@ -115,7 +116,7 @@ public class LocalProvider extends Provider implements public List<String> getDocumentTypes() { List<String> documentTypes = new ArrayList<>(); - for (SearchCluster.SchemaInfo spec : searchCluster.schemas().values()) { + for (SchemaInfo spec : searchCluster.schemas().values()) { documentTypes.add(spec.fullSchema().getDocument().getName()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 31e7bcfe2ca..82321216519 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -6,9 +6,9 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.derived.SchemaInfo; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.model.builder.UserConfigBuilder; import com.yahoo.vespa.model.builder.xml.dom.DomSearchTuningBuilder; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; @@ -243,8 +243,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); - sc.add(new SearchCluster.SchemaInfo(schema, - deployState.rankProfileRegistry())); + sc.add(new SchemaInfo(schema, deployState.rankProfileRegistry(), null, null)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java index 157c23732c7..34041217b0a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -8,9 +8,9 @@ import java.util.IdentityHashMap; import java.util.Map; /** - * @author hmusum - * <p> * File distribution config producer, delegates getting config to {@link FileDistributionConfigProvider} (one per host) + * + * @author hmusum */ public class FileDistributionConfigProducer extends AbstractConfigProducer<AbstractConfigProducer<?>> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index 618c83a3cb6..e628907068d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.ImportedFieldsConfig; @@ -33,7 +34,8 @@ public class DocumentDatabase extends AbstractConfigProducer<DocumentDatabase> i JuniperrcConfig.Producer, SummarymapConfig.Producer, SummaryConfig.Producer, - ImportedFieldsConfig.Producer { + ImportedFieldsConfig.Producer, + SchemaInfoConfig.Producer { private final String schemaName; private final DerivedConfiguration derivedCfg; @@ -57,63 +59,41 @@ public class DocumentDatabase extends AbstractConfigProducer<DocumentDatabase> i } @Override - public void getConfig(IndexInfoConfig.Builder builder) { - derivedCfg.getIndexInfo().getConfig(builder); - } + public void getConfig(IndexInfoConfig.Builder builder) { derivedCfg.getIndexInfo().getConfig(builder); } @Override - public void getConfig(IlscriptsConfig.Builder builder) { - derivedCfg.getIndexingScript().getConfig(builder); - } + public void getConfig(IlscriptsConfig.Builder builder) { derivedCfg.getIndexingScript().getConfig(builder); } @Override - public void getConfig(AttributesConfig.Builder builder) { - derivedCfg.getConfig(builder); - } + public void getConfig(AttributesConfig.Builder builder) { derivedCfg.getConfig(builder); } @Override - public void getConfig(RankProfilesConfig.Builder builder) { - derivedCfg.getRankProfileList().getConfig(builder); - } + public void getConfig(RankProfilesConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } @Override - public void getConfig(RankingExpressionsConfig.Builder builder) { - derivedCfg.getRankProfileList().getConfig(builder); - } + public void getConfig(RankingExpressionsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } @Override - public void getConfig(RankingConstantsConfig.Builder builder) { - derivedCfg.getRankProfileList().getConfig(builder); - } + public void getConfig(RankingConstantsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } @Override - public void getConfig(OnnxModelsConfig.Builder builder) { - derivedCfg.getRankProfileList().getConfig(builder); - } + public void getConfig(OnnxModelsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } @Override - public void getConfig(IndexschemaConfig.Builder builder) { - derivedCfg.getIndexSchema().getConfig(builder); - } + public void getConfig(IndexschemaConfig.Builder builder) { derivedCfg.getIndexSchema().getConfig(builder); } @Override - public void getConfig(JuniperrcConfig.Builder builder) { - derivedCfg.getJuniperrc().getConfig(builder); - } + public void getConfig(JuniperrcConfig.Builder builder) { derivedCfg.getJuniperrc().getConfig(builder); } @Override - public void getConfig(SummarymapConfig.Builder builder) { - derivedCfg.getSummaryMap().getConfig(builder); - } + public void getConfig(SummarymapConfig.Builder builder) { derivedCfg.getSummaryMap().getConfig(builder); } + @Override + public void getConfig(SummaryConfig.Builder builder) { derivedCfg.getSummaries().getConfig(builder); } @Override - public void getConfig(SummaryConfig.Builder builder) { - derivedCfg.getSummaries().getConfig(builder); - } + public void getConfig(ImportedFieldsConfig.Builder builder) { derivedCfg.getImportedFields().getConfig(builder); } @Override - public void getConfig(ImportedFieldsConfig.Builder builder) { - derivedCfg.getImportedFields().getConfig(builder); - } + public void getConfig(SchemaInfoConfig.Builder builder) { derivedCfg.getSchemaInfo().getConfig(builder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 31bd1d60c23..d5bf12df559 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; +import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.searchdefinition.DocumentOnlySchema; import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.searchdefinition.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; @@ -30,8 +33,10 @@ public class IndexedSearchCluster extends SearchCluster implements DocumentdbInfoConfig.Producer, IndexInfoConfig.Producer, + SchemaInfoConfig.Producer, IlscriptsConfig.Producer, - DispatchConfig.Producer { + DispatchConfig.Producer, + ConfigInstance.Producer { private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config. private String indexingChainName = null; @@ -44,7 +49,7 @@ public class IndexedSearchCluster extends SearchCluster // This is the document selector string as derived from the subscription tag. private String routingSelector = null; private final List<DocumentDatabase> documentDbs = new LinkedList<>(); - private final UnionConfiguration unionCfg; + private final MultipleDocumentDatabasesConfigProducer documentDbsConfigProducer; private int searchableCopies = 1; @@ -64,7 +69,7 @@ public class IndexedSearchCluster extends SearchCluster public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index) { super(parent, clusterName, index); - unionCfg = new UnionConfiguration(this, documentDbs); + documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); rootDispatch = new DispatchGroup(this); } @@ -227,22 +232,27 @@ public class IndexedSearchCluster extends SearchCluster @Override public void getConfig(IndexInfoConfig.Builder builder) { - unionCfg.getConfig(builder); + documentDbsConfigProducer.getConfig(builder); + } + + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + documentDbsConfigProducer.getConfig(builder); } @Override public void getConfig(IlscriptsConfig.Builder builder) { - unionCfg.getConfig(builder); + documentDbsConfigProducer.getConfig(builder); } @Override public void getConfig(AttributesConfig.Builder builder) { - unionCfg.getConfig(builder); + documentDbsConfigProducer.getConfig(builder); } @Override public void getConfig(RankProfilesConfig.Builder builder) { - unionCfg.getConfig(builder); + documentDbsConfigProducer.getConfig(builder); } boolean useFixedRowInDispatch() { @@ -326,21 +336,39 @@ public class IndexedSearchCluster extends SearchCluster /** * Class used to retrieve combined configuration from multiple document databases. - * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs, + * It is not a direct {@link com.yahoo.config.ConfigInstance.Producer} of those configs, * that is handled (by delegating to this) by the {@link IndexedSearchCluster} * which is the parent to this. This avoids building the config multiple times. */ - public static class UnionConfiguration - extends AbstractConfigProducer<UnionConfiguration> - implements AttributesConfig.Producer { + public static class MultipleDocumentDatabasesConfigProducer + extends AbstractConfigProducer<MultipleDocumentDatabasesConfigProducer> + implements AttributesConfig.Producer, + IndexInfoConfig.Producer, + IlscriptsConfig.Producer, + SchemaInfoConfig.Producer, + RankProfilesConfig.Producer { private final List<DocumentDatabase> docDbs; + private MultipleDocumentDatabasesConfigProducer(AbstractConfigProducer<?> parent, List<DocumentDatabase> docDbs) { + super(parent, "union"); + this.docDbs = docDbs; + } + + @Override public void getConfig(IndexInfoConfig.Builder builder) { for (DocumentDatabase docDb : docDbs) { docDb.getConfig(builder); } } + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + for (DocumentDatabase docDb : docDbs) { + docDb.getConfig(builder); + } + } + + @Override public void getConfig(IlscriptsConfig.Builder builder) { for (DocumentDatabase docDb : docDbs) { docDb.getConfig(builder); @@ -354,16 +382,13 @@ public class IndexedSearchCluster extends SearchCluster } } + @Override public void getConfig(RankProfilesConfig.Builder builder) { for (DocumentDatabase docDb : docDbs) { docDb.getConfig(builder); } } - private UnionConfiguration(AbstractConfigProducer<?> parent, List<DocumentDatabase> docDbs) { - super(parent, "union"); - this.docDbs = docDbs; - } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 0f9ea7fbaf6..d7ce64d1d32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -2,10 +2,9 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.derived.SchemaInfo; import com.yahoo.searchdefinition.derived.SummaryMap; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; @@ -19,7 +18,6 @@ import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.config.model.producer.AbstractConfigProducer; -import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -35,7 +33,8 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster implements DocumentdbInfoConfig.Producer, IndexInfoConfig.Producer, - IlscriptsConfig.Producer { + IlscriptsConfig.Producer, + SchemaInfoConfig.Producer { private final String clusterName; private int index; @@ -113,7 +112,7 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster } protected void addRankProfilesConfig(String schemaName, DocumentdbInfoConfig.Documentdb.Builder docDbBuilder) { - for (RankProfileInfo rankProfile : schemas().get(schemaName).rankProfiles().values()) { + for (SchemaInfo.RankProfileInfo rankProfile : schemas().get(schemaName).rankProfiles().values()) { var rankProfileConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); rankProfileConfig.name(rankProfile.name()); rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); @@ -153,12 +152,19 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster public abstract void defaultDocumentsConfig(); public abstract DerivedConfiguration getSchemaConfig(); + // TODO: The get methods below should be moved to StreamingSearchCluster + @Override public void getConfig(IndexInfoConfig.Builder builder) { if (getSchemaConfig() != null) getSchemaConfig().getIndexInfo().getConfig(builder); } @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + if (getSchemaConfig() != null) getSchemaConfig().getSchemaInfo().getConfig(builder); + } + + @Override public void getConfig(IlscriptsConfig.Builder builder) { if (getSchemaConfig() != null) getSchemaConfig().getIndexingScript().getConfig(builder); } @@ -195,51 +201,4 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster } } - public static final class SchemaInfo { - - private final Schema schema; - - // Info about profiles needed in memory after build. - // The rank profile registry itself is not kept around due to its size. - private final Map<String, RankProfileInfo> rankProfiles; - - public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry) { - this.schema = schema; - this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); - } - - public String name() { return schema.getName(); } - public Schema fullSchema() { return schema; } - public Map<String, RankProfileInfo> rankProfiles() { return rankProfiles; } - - private Map<String, RankProfileInfo> toRankProfiles(Collection<RankProfile> rankProfiles) { - Map<String, RankProfileInfo> rankProfileInfos = new LinkedHashMap<>(); - rankProfiles.forEach(profile -> rankProfileInfos.put(profile.name(), new RankProfileInfo(profile))); - return rankProfileInfos; - } - - } - - /** A store of a *small* (in memory) amount of rank profile info. */ - public static final class RankProfileInfo { - - private final String name; - private final boolean hasSummaryFeatures; - private final boolean hasRankFeatures; - private final Map<Reference, TensorType> inputs; - - public RankProfileInfo(RankProfile profile) { - this.name = profile.name(); - this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); - this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); - this.inputs = profile.inputs(); - } - - public String name() { return name; } - public boolean hasSummaryFeatures() { return hasSummaryFeatures; } - public boolean hasRankFeatures() { return hasRankFeatures; } - public Map<Reference, TensorType> inputs() { return inputs; } - - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 3d0092b093a..df266ab77dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -87,9 +87,7 @@ public class StreamingSearchCluster extends SearchCluster implements } @Override - public DerivedConfiguration getSchemaConfig() { - return schemaConfig; - } + public DerivedConfiguration getSchemaConfig() { return schemaConfig; } @Override public void defaultDocumentsConfig() { } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index bfa135de90e..6abe0e763cb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -31,44 +31,44 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); - SummaryMap summaryMap=new SummaryMap(schema); + SummaryMap summaryMap = new SummaryMap(schema); - Iterator transforms=summaryMap.resultTransformIterator(); - FieldResultTransform transform = (FieldResultTransform)transforms.next(); + Iterator<FieldResultTransform> transforms = summaryMap.resultTransforms().values().iterator(); + FieldResultTransform transform = transforms.next(); assertEquals("dyndesc", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("dynlong", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("dyndesc2", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("measurement", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("rankfeatures", transform.getFieldName()); assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("summaryfeatures", transform.getFieldName()); assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("popsiness", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("popularity", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("access", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); assertFalse(transforms.hasNext()); } @@ -84,22 +84,22 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { true, false, Set.of()); SummaryMap summaryMap = new SummaryMap(schema); - Iterator transforms = summaryMap.resultTransformIterator(); + Iterator<FieldResultTransform> transforms = summaryMap.resultTransforms().values().iterator(); - FieldResultTransform transform = (FieldResultTransform)transforms.next(); + FieldResultTransform transform = transforms.next(); assertEquals(fieldName, transform.getFieldName()); assertEquals(SummaryTransform.GEOPOS, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("rankfeatures", transform.getFieldName()); assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("summaryfeatures", transform.getFieldName()); assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("location_zcurve", transform.getFieldName()); assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index 9a36ef90cd7..fd5caeb6b6d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -37,7 +37,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { "}"); Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); - assertEquals(SummaryClassField.Type.RAW, summary.getField("raw_field").getType()); + assertEquals(SummaryClassField.Type.RAW, summary.fields().get("raw_field").getType()); } @Test @@ -52,7 +52,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { "}"); Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); - assertEquals(SummaryClassField.Type.DATA, summary.getField("raw_field").getType()); + assertEquals(SummaryClassField.Type.DATA, summary.fields().get("raw_field").getType()); } @Test @@ -61,11 +61,11 @@ public class SummaryTestCase extends AbstractSchemaTestCase { SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); assertEquals("default", summary.getName()); - Iterator<SummaryClassField> fields = summary.fieldIterator(); + Iterator<SummaryClassField> fields = summary.fields().values().iterator(); SummaryClassField field; - assertEquals(13, summary.getFieldCount()); + assertEquals(13, summary.fields().size()); field = fields.next(); assertEquals("exactemento", field.getName()); @@ -125,12 +125,12 @@ public class SummaryTestCase extends AbstractSchemaTestCase { Schema adSchema = buildCampaignAdModel(); SummaryClass defaultClass = new SummaryClass(adSchema, adSchema.getSummary("default"), new BaseDeployLogger()); - assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.getField("campaign_ref").getType()); - assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.getField("other_campaign_ref").getType()); + assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.fields().get("campaign_ref").getType()); + assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.fields().get("other_campaign_ref").getType()); SummaryClass myClass = new SummaryClass(adSchema, adSchema.getSummary("my_summary"), new BaseDeployLogger()); - assertNull(myClass.getField("campaign_ref")); - assertEquals(SummaryClassField.Type.LONGSTRING, myClass.getField("other_campaign_ref").getType()); + assertNull(myClass.fields().get("campaign_ref")); + assertEquals(SummaryClassField.Type.LONGSTRING, myClass.fields().get("other_campaign_ref").getType()); } private static Schema buildCampaignAdModel() throws ParseException { diff --git a/configserver/pom.xml b/configserver/pom.xml index 99307d1b2b8..131e4503f6d 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -189,11 +189,39 @@ <scope>provided</scope> </dependency> <dependency> + <!-- Do not remove, as long as this is provided by jdisc and configserver uses jersey-client --> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <!-- Do not remove, as long as this is provided by jdisc and configserver uses jersey-client --> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <scope>provided</scope> </dependency> <dependency> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-json-provider</artifactId> + <exclusions> + <exclusion> + <!-- Conflicts with javax.activation:javax.activation-api:1.2.0, which is "exported" via jdisc_core. --> + <groupId>jakarta.activation</groupId> + <artifactId>jakarta.activation-api</artifactId> + </exclusion> + <exclusion> + <!-- Conflicts with javax.xml.bind:jaxb-api:2.3, which is "exported" via jdisc_core.--> + <groupId>jakarta.xml.bind</groupId> + <artifactId>jakarta.xml.bind-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> </dependency> @@ -213,6 +241,52 @@ <version>${project.version}</version> </dependency> + <!-- Jersey, needed by orchestrator --> + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <scope>provided</scope> <!-- TODO: Vespa 8: Set to compile if we get rid of the javax.ws.rs-api bundle --> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-client</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-server</artifactId> + <exclusions> + <exclusion> + <groupId>org.glassfish.jersey.media</groupId> + <artifactId>jersey-media-jaxb</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.ext</groupId> + <artifactId>jersey-proxy-client</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.media</groupId> + <artifactId>jersey-media-json-jackson</artifactId> + <exclusions> + <!-- Prevent embedding deps provided by jdisc --> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <!-- Not needed by configserver, but by controller. Also pulls in mimepull. --> + <groupId>org.glassfish.jersey.media</groupId> + <artifactId>jersey-media-multipart</artifactId> + </dependency> + <!-- Jersey END --> + </dependencies> <build> <plugins> diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index 9da16744eec..be8ba669ec0 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -7,9 +7,10 @@ import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.DistributionConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; /** - * Lets a lazily initialised DocumentAccess that forwards to a MessageBusDocumentAccess be injected in containers. + * Lets a lazily initialised DocumentAccess forwarding to a real MessageBusDocumentAccess be injected in containers. * * @author jonmv */ diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java index c708e87e6c5..7c5bebc47e8 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -20,8 +20,8 @@ import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.yolean.concurrent.Memoized; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -37,7 +37,8 @@ public class VespaDocumentAccess extends DocumentAccess { private final MessageBusParams parameters; - private final Memoized<DocumentAccess, RuntimeException> delegate; + private final AtomicReference<DocumentAccess> delegate = new AtomicReference<>(); + private boolean shutDown = false; VespaDocumentAccess(DocumentmanagerConfig documentmanagerConfig, String slobroksConfigId, @@ -50,11 +51,19 @@ public class VespaDocumentAccess extends DocumentAccess { this.parameters.setDocumentmanagerConfig(documentmanagerConfig); this.parameters.getRPCNetworkParams().setSlobrokConfigId(slobroksConfigId); this.parameters.getMessageBusParams().setMessageBusConfig(messagebusConfig); - this.delegate = new Memoized<>(() -> new MessageBusDocumentAccess(parameters), DocumentAccess::shutdown); } public DocumentAccess delegate() { - return delegate.get(); + DocumentAccess access = delegate.getAcquire(); + return access != null ? access : delegate.updateAndGet(value -> { + if (value != null) + return value; + + if (shutDown) + throw new IllegalStateException("This document access has been shut down"); + + return new MessageBusDocumentAccess(parameters); + }); } @Override @@ -63,7 +72,14 @@ public class VespaDocumentAccess extends DocumentAccess { } void protectedShutdown() { - delegate.close(); + delegate.updateAndGet(access -> { + super.shutdown(); + shutDown = true; + if (access != null) + access.shutdown(); + + return null; + }); } @Override diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java index 8b3b2664cd1..cb28807ac73 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerHolder.java @@ -7,9 +7,6 @@ import com.yahoo.messagebus.network.NetworkMultiplexer; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.shared.NullNetwork; -import com.yahoo.yolean.concurrent.Memoized; - -import java.util.concurrent.atomic.AtomicReference; /** * Holds a reference to a singleton {@link NetworkMultiplexer}. @@ -18,17 +15,21 @@ import java.util.concurrent.atomic.AtomicReference; */ public class NetworkMultiplexerHolder extends AbstractComponent { - private final AtomicReference<RPCNetworkParams> params = new AtomicReference<>(); - private final Memoized<NetworkMultiplexer, RuntimeException> net = new Memoized<>(() -> NetworkMultiplexer.shared(newNetwork(params.get())), - NetworkMultiplexer::disown); + private final Object monitor = new Object(); + private boolean destroyed = false; + private NetworkMultiplexer net; /** Get the singleton RPCNetworkAdapter, creating it if this hasn't yet been done. */ public NetworkMultiplexer get(RPCNetworkParams params) { - this.params.set(params); - return net.get(); + synchronized (monitor) { + if (destroyed) + throw new IllegalStateException("Component already destroyed"); + + return net = net != null ? net : NetworkMultiplexer.shared(newNetwork(params)); + } } - private static Network newNetwork(RPCNetworkParams params) { + private Network newNetwork(RPCNetworkParams params) { return params.getSlobroksConfig() != null && params.getSlobroksConfig().slobrok().isEmpty() ? new NullNetwork() // For LocalApplication, test setup. : new RPCNetwork(params); @@ -36,7 +37,13 @@ public class NetworkMultiplexerHolder extends AbstractComponent { @Override public void deconstruct() { - net.close(); + synchronized (monitor) { + if (net != null) { + net.disown(); + net = null; + } + destroyed = true; + } } } diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 3045247a6d3..91a3181cb68 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -29,7 +29,6 @@ import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.yolean.concurrent.Memoized; import java.util.HashMap; import java.util.Map; @@ -55,7 +54,9 @@ public final class SessionCache extends AbstractComponent { private static final Logger log = Logger.getLogger(SessionCache.class.getName()); - private final Memoized<SharedMessageBus, RuntimeException> messageBus; + private final Object monitor = new Object(); + private Supplier<SharedMessageBus> messageBuses; + private SharedMessageBus messageBus; private final Object intermediateLock = new Object(); private final Map<String, SharedIntermediateSession> intermediates = new HashMap<>(); @@ -95,18 +96,24 @@ public final class SessionCache extends AbstractComponent { public SessionCache(Supplier<NetworkMultiplexer> net, ContainerMbusConfig containerMbusConfig, MessagebusConfig messagebusConfig, Protocol protocol) { - this.messageBus = new Memoized<>(() -> createSharedMessageBus(net.get(), containerMbusConfig, messagebusConfig, protocol), - SharedMessageBus::release); + this.messageBuses = () -> createSharedMessageBus(net.get(), containerMbusConfig, messagebusConfig, protocol); } @Override public void deconstruct() { - messageBus.close(); + synchronized (monitor) { + messageBuses = () -> { throw new IllegalStateException("Session cache already deconstructed"); }; + + if (messageBus != null) + messageBus.release(); + } } // Lazily create shared message bus. private SharedMessageBus bus() { - return messageBus.get(); + synchronized (monitor) { + return messageBus = messageBus != null ? messageBus : messageBuses.get(); + } } private static SharedMessageBus createSharedMessageBus(NetworkMultiplexer net, diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 54a3b3a0f36..fb1740d0e5e 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -8060,6 +8060,7 @@ "methods": [ "public void <init>(com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig)", "public void <init>(java.util.List, java.util.Map)", + "public java.util.List schemas()", "public com.yahoo.search.schema.SchemaInfo$Session newSession(com.yahoo.search.Query)", "public static com.yahoo.search.schema.SchemaInfo empty()", "public boolean equals(java.lang.Object)", @@ -8098,6 +8099,7 @@ "public static com.yahoo.search.searchchain.Execution$Context createContextStub()", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts)", + "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.schema.SchemaInfo)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts, com.yahoo.language.Linguistics)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.Linguistics)", diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 5b2df3485c8..3953190eac4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -39,6 +39,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; +import java.util.stream.Collectors; import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.STREAMING; @@ -58,10 +59,10 @@ public class ClusterSearcher extends Searcher { private final String searchClusterName; // The set of document types contained in this search cluster - private final Set<String> documentTypes; + private final Set<String> schemas; - // Mapping from rank profile names to document types containing them - private final Map<String, Set<String>> rankProfiles = new HashMap<>(); + // Mapping from rank profile names to schemas containing them + private final Map<String, Set<String>> rankProfilesz = new HashMap<>(); private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds @@ -83,7 +84,7 @@ public class ClusterSearcher extends Searcher { int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); - documentTypes = new LinkedHashSet<>(); + schemas = new LinkedHashSet<>(); maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT); maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); @@ -92,14 +93,8 @@ public class ClusterSearcher extends Searcher { .com().yahoo().prelude().fastsearch().FastSearcher().docsum() .defaultclass()); - for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { - String docTypeName = docDb.name(); - documentTypes.add(docTypeName); - - for (DocumentdbInfoConfig.Documentdb.Rankprofile profile : docDb.rankprofile()) { - addValidRankProfile(profile.name(), docTypeName); - } - } + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) + schemas.add(docDb.name()); String uniqueServerId = UUID.randomUUID().toString(); if (searchClusterConfig.indexingmode() == STREAMING) { @@ -159,8 +154,8 @@ public class ClusterSearcher extends Searcher { } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(Set<String> documentTypes, VespaBackEndSearcher searcher, Executor executor) { - this.documentTypes = documentTypes; + ClusterSearcher(Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) { + this.schemas = schemas; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; @@ -168,19 +163,8 @@ public class ClusterSearcher extends Searcher { this.executor = executor; } /** Do not use, for internal testing purposes only. **/ - ClusterSearcher(Set<String> documentTypes) { - this(documentTypes, null, null); - } - - void addValidRankProfile(String profileName, String docTypeName) { - if (!rankProfiles.containsKey(profileName)) { - rankProfiles.put(profileName, new HashSet<>()); - } - rankProfiles.get(profileName).add(docTypeName); - } - - void setValidRankProfile(String profileName, Set<String> documentTypes) { - rankProfiles.put(profileName, documentTypes); + ClusterSearcher(Set<String> schemas) { + this(schemas, null, null); } /** @@ -192,49 +176,54 @@ public class ClusterSearcher extends Searcher { * probably not reasonable. * * @param query query - * @param docTypes set of requested doc types for this query - * @return null if request rank profile is ok for the requested - * doc types, a result with error message if not. + * @param schemas set of requested schemas for this query + * @return null if requested rank profile is ok for the requested + * schemas, a result with error message if not. */ // TODO: This should be in a separate searcher - private Result checkValidRankProfiles(Query query, Set<String> docTypes) { + // TODO Vespa 8: This should simply fail if the specified profile isn't present in all schemas + private Result checkValidRankProfiles(Query query, Set<String> schemas, Execution.Context context) { String rankProfile = query.getRanking().getProfile(); - Set<String> invalidInDocTypes = null; - Set<String> rankDocTypes = rankProfiles.get(rankProfile); + Set<String> invalidInSchemas = null; + Set<String> schemasHavingProfile = schemasHavingProfile(rankProfile, context); - if (rankDocTypes == null) { - // rank profile does not exist in any document type - invalidInDocTypes = docTypes; + if (schemasHavingProfile.isEmpty()) { + invalidInSchemas = schemas; } - else if (docTypes.size() == 1) { - // one document type, fails if invalid rank profile - if (!rankDocTypes.contains(docTypes.iterator().next())) { - invalidInDocTypes = docTypes; - } + else if (schemas.size() == 1) { + if ( ! schemasHavingProfile.containsAll(schemas)) + invalidInSchemas = schemas; } else { - // multiple document types, only fail when restricting doc types + // multiple schemas, only fail when restricting doc types Set<String> restrict = query.getModel().getRestrict(); Set<String> sources = query.getModel().getSources(); boolean validate = restrict != null && !restrict.isEmpty(); validate = validate || sources != null && !sources.isEmpty(); - if (validate && !rankDocTypes.containsAll(docTypes)) { - invalidInDocTypes = new HashSet<>(docTypes); - invalidInDocTypes.removeAll(rankDocTypes); + if (validate && !schemasHavingProfile.containsAll(schemas)) { + invalidInSchemas = new HashSet<>(schemas); + invalidInSchemas.removeAll(schemasHavingProfile); } } - if (invalidInDocTypes != null && !invalidInDocTypes.isEmpty()) { - String plural = invalidInDocTypes.size() > 1 ? "s" : ""; + if (invalidInSchemas != null && !invalidInSchemas.isEmpty()) { + String plural = invalidInSchemas.size() > 1 ? "s" : ""; return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested rank profile '" + rankProfile + "' is undefined for document type" + plural + " '" + - String.join(", ", invalidInDocTypes) + "'")); + String.join(", ", invalidInSchemas) + "'")); } return null; } + private Set<String> schemasHavingProfile(String profile, Execution.Context context) { + return context.schemaInfo().schemas().stream() + .filter(schema -> schema.rankProfiles().containsKey(profile)) + .map(schema -> schema.name()) + .collect(Collectors.toSet()); + } + @Override public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); @@ -292,12 +281,12 @@ public class ClusterSearcher extends Searcher { } private Result doSearch(Searcher searcher, Query query, Execution execution) { - if (documentTypes.size() > 1) { + if (schemas.size() > 1) { return searchMultipleDocumentTypes(searcher, query, execution); } else { - String docType = documentTypes.iterator().next(); + String docType = schemas.iterator().next(); - Result invalidRankProfile = checkValidRankProfiles(query, documentTypes); + Result invalidRankProfile = checkValidRankProfiles(query, schemas, execution.context()); if (invalidRankProfile != null) { return invalidRankProfile; } @@ -320,12 +309,12 @@ public class ClusterSearcher extends Searcher { } private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) { - Set<String> docTypes = resolveDocumentTypes(query, execution.context().getIndexFacts()); + Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); - Result invalidRankProfile = checkValidRankProfiles(query, docTypes); + Result invalidRankProfile = checkValidRankProfiles(query, schemas, execution.context()); if (invalidRankProfile != null) return invalidRankProfile; - List<Query> queries = createQueries(query, docTypes); + List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { return searcher.search(queries.get(0), execution); } else { @@ -357,13 +346,13 @@ public class ClusterSearcher extends Searcher { } } - Set<String> resolveDocumentTypes(Query query, IndexFacts indexFacts) { + Set<String> resolveSchemas(Query query, IndexFacts indexFacts) { Set<String> restrict = query.getModel().getRestrict(); if (restrict == null || restrict.isEmpty()) { Set<String> sources = query.getModel().getSources(); return (sources == null || sources.isEmpty()) - ? documentTypes - : new HashSet<>(indexFacts.newSession(sources, Collections.emptyList(), documentTypes).documentTypes()); + ? schemas + : new HashSet<>(indexFacts.newSession(sources, Collections.emptyList(), schemas).documentTypes()); } else { return filterValidDocumentTypes(restrict); } @@ -372,7 +361,7 @@ public class ClusterSearcher extends Searcher { private Set<String> filterValidDocumentTypes(Collection<String> restrict) { Set<String> retval = new LinkedHashSet<>(); for (String docType : restrict) { - if (docType != null && documentTypes.contains(docType)) { + if (docType != null && schemas.contains(docType)) { retval.add(docType); } } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index 2d1d391640f..4daf110fc54 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -55,6 +55,9 @@ public class SchemaInfo { this.clusters = Map.copyOf(clusters); } + /** Returns all schemas configured in this application. */ + public List<Schema> schemas() { return schemas; } + public Session newSession(Query query) { return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index aba72cb3404..94ff9745ea6 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -165,6 +165,11 @@ public class Execution extends com.yahoo.processing.execution.Execution { } /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ + public static Context createContextStub(SchemaInfo schemaInfo) { + return createContextStub(null, null, schemaInfo, null); + } + + /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts) { return createContextStub(searchChainRegistry, indexFacts, SchemaInfo.empty(), null); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index 54874dbee3e..3e44b02618e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -68,7 +68,7 @@ public class ExecutionFactory extends AbstractComponent { this.executor = executor != null ? executor : Executors.newSingleThreadExecutor(); } - /** @deprecated pass documentDbInfo */ + /** @deprecated pass SchemaInfoConfig */ @Deprecated public ExecutionFactory(ChainsConfig chainsConfig, IndexInfoConfig indexInfo, diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def index e5b14db9b4e..2da6d621973 100644 --- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -4,16 +4,14 @@ namespace=search.config ## The name of this schema schema[].name string -## The id of the summary class -# schema[].summaryclass[].id int ## The name of the summary class -# schema[].summaryclass[].name string +schema[].summaryclass[].name string ## The name of a field in the summary class -# schema[].summaryclass[].fields[].name string +schema[].summaryclass[].fields[].name string ## The type of a field in the summary class -# schema[].summaryclass[].fields[].type string +schema[].summaryclass[].fields[].type string ## Whether this field is a dynamic snippet -# schema[].summaryclass[].fields[].dynamic bool default=false +schema[].summaryclass[].fields[].dynamic bool default=false ## Information about rank profiles schema[].rankprofile[].name string diff --git a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def index 0fa448e1572..4528475697b 100644 --- a/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def +++ b/container-search/src/main/resources/configdefinitions/prelude.fastsearch.documentdb-info.def @@ -1,7 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=prelude.fastsearch -## The name of the search definition that this document database info applies to +## The name of the schema/document database documentdb[].name string ## The id of the summary class diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 801f7da1939..2d62e4c0154 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -22,6 +22,9 @@ import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.result.Hit; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; import org.junit.Test; @@ -81,7 +84,7 @@ public class ClusterSearcherTestCase { } private Set<String> resolve(ClusterSearcher searcher, String query) { - return searcher.resolveDocumentTypes(new Query("?query=hello" + query), createIndexFacts()); + return searcher.resolveSchemas(new Query("?query=hello" + query), createIndexFacts()); } @Test @@ -267,9 +270,15 @@ public class ClusterSearcherTestCase { new MyMockSearcher(expectAttributePrefetch), new InThreadExecutorService()); try { - cluster.setValidRankProfile("default", documentTypes); - cluster.addValidRankProfile("testprofile", "type1"); - return new Execution(cluster, Execution.Context.createContextStub()); + List<Schema> schemas = new ArrayList<>(); + for (String docType : docTypesList) { + var schemaBuilder = new Schema.Builder(docType); + schemaBuilder.add(new RankProfile.Builder("default").build()); + if (docType.equals("type1")) + schemaBuilder.add(new RankProfile.Builder("testprofile").build()); + schemas.add(schemaBuilder.build()); + } + return new Execution(cluster, Execution.Context.createContextStub(new SchemaInfo(schemas, Map.of()))); } finally { cluster.deconstruct(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java index 8123b6f2ce6..dc8b22ac32d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java @@ -2,13 +2,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.organization; -import com.yahoo.vespa.hosted.controller.api.integration.jira.JiraIssue; - -import java.io.InputStream; import java.time.Duration; -import java.util.List; import java.util.Optional; -import java.util.function.Supplier; /** * @author jonmv @@ -24,22 +19,12 @@ public interface IssueHandler { IssueId file(Issue issue); /** - * Returns all open issues similar to the given. - * - * @param issue The issue to search for; relevant fields are the summary and the owner (propertyId). - * @return All open, similar issues. - */ - List<IssueInfo> findAllBySimilarity(Issue issue); - - /** * Returns the ID of this issue, if it exists and is open, based on a similarity search. * * @param issue The issue to search for; relevant fields are the summary and the owner (propertyId). * @return ID of the issue, if it is found. */ - default Optional<IssueId> findBySimilarity(Issue issue) { - return findAllBySimilarity(issue).stream().findFirst().map(IssueInfo::id); - } + Optional<IssueId> findBySimilarity(Issue issue); /** * Update the description of the issue with the given ID. @@ -123,8 +108,4 @@ public interface IssueHandler { * @throws RuntimeException exception if project not found */ ProjectInfo projectInfo(String projectKey); - - /** Upload an attachment to the issue, with indicated filename, from the given input stream. */ - void addAttachment(IssueId id, String filename, Supplier<InputStream> contentAsStream); - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueInfo.java deleted file mode 100644 index 52c022bebdf..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueInfo.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.yahoo.vespa.hosted.controller.api.integration.organization; - -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; - -import java.time.Instant; -import java.util.Optional; - -/** - * Information about a stored issue. - * - * @author jonmv - */ -public class IssueInfo { - - private final IssueId id; - private final Instant updated; - private final Status status; - private final User assignee; - - public IssueInfo(IssueId id, Instant updated, Status status, User assignee) { - this.id = id; - this.updated = updated; - this.status = status; - this.assignee = assignee; - } - - public IssueId id() { - return id; - } - - public Instant updated() { - return updated; - } - - public Status status() { - return status; - } - - public Optional<User> assignee() { - return Optional.ofNullable(assignee); - } - - - public enum Status { - - toDo("To Do"), - inProgress("In Progress"), - done("Done"), - noCategory("No Category"); - - private final String value; - - Status(String value) { this.value = value; } - - public static Status fromValue(String value) { - for (Status status : Status.values()) - if (status.value.equals(value)) - return status; - throw new IllegalArgumentException(value + " is not a valid status."); - } - - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java index a62f43d1cf5..257d2ff5e67 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.organization; import com.google.inject.Inject; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueInfo.Status; -import java.io.InputStream; import java.net.URI; import java.time.Clock; import java.time.Duration; @@ -16,7 +14,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -27,7 +24,6 @@ public class MockIssueHandler implements IssueHandler { private final Clock clock; private final AtomicLong counter = new AtomicLong(); private final Map<IssueId, MockIssue> issues = new HashMap<>(); - private final Map<IssueId, Map<String, InputStream>> attachments = new HashMap<>(); private final Map<String, ProjectInfo> projects = new HashMap<>(); @Inject @@ -49,14 +45,11 @@ public class MockIssueHandler implements IssueHandler { } @Override - public List<IssueInfo> findAllBySimilarity(Issue issue) { + public Optional<IssueId> findBySimilarity(Issue issue) { return issues.entrySet().stream() - .filter(entry -> entry.getValue().issue.summary().equals(issue.summary())) - .map(entry -> new IssueInfo(entry.getKey(), - entry.getValue().updated, - entry.getValue().isOpen() ? Status.toDo : Status.done, - entry.getValue().assignee)) - .collect(Collectors.toList()); + .filter(entry -> entry.getValue().issue.summary().equals(issue.summary())) + .findFirst() + .map(Map.Entry::getKey); } @Override @@ -125,11 +118,6 @@ public class MockIssueHandler implements IssueHandler { return projects.get(projectKey); } - @Override - public void addAttachment(IssueId id, String filename, Supplier<InputStream> contentAsStream) { - attachments.computeIfAbsent(id, __ -> new HashMap<>()).put(filename, contentAsStream.get()); - } - public MockIssueHandler close(IssueId issueId) { issues.get(issueId).open = false; touch(issueId); diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java index 8c858437ad7..f155cbbce07 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; -import com.yahoo.yolean.Exceptions; - import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -53,26 +51,26 @@ public class MultiPartStreamer { /** Adds the given data as a named part in this, using the given content type. */ public MultiPartStreamer addData(String name, String type, String data) { - return addData(name, type, null, () -> asStream(data)); + streams.add(() -> separator(name, type)); + streams.add(() -> asStream(data)); + + return this; } /** Adds the given data as a named part in this, using the {@code "application/octet-stream" content type}. */ public MultiPartStreamer addBytes(String name, byte[] bytes) { - return addData(name, "application/octet-stream", null, () -> new ByteArrayInputStream(bytes)); - } - - /** Adds the given data as a named part in this, using the given content type. */ - public MultiPartStreamer addData(String name, String type, String filename, Supplier<InputStream> data) { - streams.add(() -> separator(name, filename, type)); - streams.add(data); + streams.add(() -> separator(name, "application/octet-stream")); + streams.add(() -> new ByteArrayInputStream(bytes)); return this; } /** Adds the contents of the file at the given path as a named part in this. */ public MultiPartStreamer addFile(String name, Path path) { - String type = Exceptions.uncheck(() -> Files.probeContentType(path)); - return addData(name, type != null ? type : "application/octet-stream", path.getFileName().toString(), () -> asStream(path)); + streams.add(() -> separator(name, path)); + streams.add(() -> asStream(path)); + + return this; } /** @@ -109,15 +107,16 @@ public class MultiPartStreamer { } /** Returns the separator to put between one part and the next, when this is a string. */ - private InputStream separator(String name, String filename, String contentType) { - return asStream(disposition(name) + (filename == null ? "" : "; filename=\"" + filename + "\"") + type(contentType)); + private InputStream separator(String name, String contentType) { + return asStream(disposition(name) + type(contentType)); } /** Returns the separator to put between one part and the next, when this is a file. */ private InputStream separator(String name, Path path) { try { String contentType = Files.probeContentType(path); - return separator(name, path.getFileName().toString(), contentType != null ? contentType : "application/octet-stream"); + return asStream(disposition(name) + "; filename=\"" + path.getFileName() + "\"" + + type(contentType != null ? contentType : "application/octet-stream")); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java b/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java index 6a76ef65082..68741d6d509 100644 --- a/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java +++ b/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java @@ -79,9 +79,9 @@ public abstract class AbstractHttpClient implements HttpClient { .asURI()) .build(); builder.headers.forEach((name, values) -> values.forEach(value -> request.setHeader(name, value))); + request.setEntity(builder.entity); - try (HttpEntity entity = builder.entity.get()) { - request.setEntity(entity); + try { try { return handler.apply(execute(request, contextWithTimeout(builder)), request); } @@ -90,15 +90,16 @@ public abstract class AbstractHttpClient implements HttpClient { throw RetryException.wrap(e, request); } } - catch (IOException e) { - throw new UncheckedIOException("failed closing request entity", e); - } catch (RetryException e) { if (thrown == null) thrown = e.getCause(); else thrown.addSuppressed(e.getCause()); + if (builder.entity != null && ! builder.entity.isRepeatable()) { + log.log(WARNING, "Cannot retry " + request + " as entity is not repeatable"); + break; + } log.log(FINE, e.getCause(), () -> request + " failed; will retry"); } } @@ -151,7 +152,7 @@ public abstract class AbstractHttpClient implements HttpClient { private HttpURL.Query query = Query.empty(); private List<Supplier<Query>> dynamicQuery = new ArrayList<>(); private Map<String, List<String>> headers = new LinkedHashMap<>(); - private Supplier<HttpEntity> entity = () -> null; + private HttpEntity entity; private RequestConfig config = HttpClient.defaultRequestConfig; private ResponseVerifier verifier = HttpClient.throwOnError; private ExceptionHandler catcher = HttpClient.retryAll; @@ -177,7 +178,7 @@ public abstract class AbstractHttpClient implements HttpClient { } @Override - public RequestBuilder body(Supplier<HttpEntity> entity) { + public RequestBuilder body(HttpEntity entity) { this.entity = requireNonNull(entity); return this; } diff --git a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java index 16a419bf324..b41b16c25be 100644 --- a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java +++ b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java @@ -78,13 +78,7 @@ public interface HttpClient extends Closeable { RequestBuilder body(byte[] json); /** Sets the request body. */ - default RequestBuilder body(HttpEntity entity) { - if (entity.isRepeatable()) return body(() -> entity); - throw new IllegalArgumentException("entitiy must be repeatable, or a supplier must be used"); - } - - /** Sets the request body. */ - RequestBuilder body(Supplier<HttpEntity> entity); + RequestBuilder body(HttpEntity entity); /** Sets query parameters without a value, like {@code ?debug&recursive}. */ default RequestBuilder emptyParameters(String... keys) { diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/DefaultHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/DefaultHttpClientBuilder.java deleted file mode 100644 index 8ad9d63cd1a..00000000000 --- a/http-utils/src/main/java/ai/vespa/util/http/hc5/DefaultHttpClientBuilder.java +++ /dev/null @@ -1,49 +0,0 @@ -package ai.vespa.util.http.hc5; - -import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; -import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder; -import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.HttpHeaders; -import org.apache.hc.core5.util.Timeout; - -import javax.net.ssl.SSLContext; -import java.time.Duration; -import java.util.Map; -import java.util.function.Supplier; - -/** - * Like {@link VespaHttpClientBuilder}, but with standard TLS based on provided SSL context. - * - * @author jonmv - */ -public class DefaultHttpClientBuilder { - - public static final Duration connectTimeout = Duration.ofSeconds(5); - public static final Duration socketTimeout = Duration.ofSeconds(5); - - private DefaultHttpClientBuilder() { } - - public static HttpClientBuilder create(SSLContext sslContext, String userAgent) { - return create(() -> sslContext, userAgent); - } - - /** Creates an HTTP client builder with the given SSL context, and using the provided timeouts for requests where config is not overridden. */ - public static HttpClientBuilder create(Supplier<SSLContext> sslContext, String userAgent) { - return HttpClientBuilder.create() - .setConnectionManager(PoolingHttpClientConnectionManagerBuilder - .create() - .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder - .create() - .setSslContext(sslContext.get()) - .build()) - .build()) - .setUserAgent(userAgent) - .disableCookieManagement() - .disableAutomaticRetries() - .disableAuthCaching(); - } - -} diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index eea9610c7ab..38b463d6bc2 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -232,7 +232,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) const int64_t firstPos(currPos); Packet packet = readPacket(transLog, all, TARGET_PACKET_SIZE, allowTruncate); if (!packet.empty()) { - _sz += packet.size(); + set_size(size() + packet.size()); const SerialNum firstSerial = packet.range().from(); if (currPos == _headerLen) { _range.from(firstSerial); @@ -384,7 +384,7 @@ DomainPart::commit(const SerializedChunk & serialized) int64_t firstPos(byteSize()); assert(_range.to() < range.to()); - _sz += serialized.getNumEntries(); + set_size(size() + serialized.getNumEntries()); _range.to(range.to()); if (_range.from() == 0) { _range.from(range.from()); diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h index ea5290c433b..fd84040228d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h @@ -37,7 +37,7 @@ public: return _syncedSerial; } - size_t size() const { return _sz; } + size_t size() const noexcept { return _sz.load(std::memory_order_relaxed); } size_t byteSize() const { return _byteSize.load(std::memory_order_acquire); } @@ -51,6 +51,7 @@ private: void write(FastOS_FileInterface &file, SerialNumRange range, vespalib::ConstBufferRef buf); void writeHeader(const common::FileHeaderContext &fileHeaderContext); + void set_size(size_t sz) noexcept { _sz.store(sz, std::memory_order_relaxed); } class SkipInfo { @@ -72,7 +73,7 @@ private: std::mutex _lock; std::mutex _fileLock; SerialNumRange _range; - size_t _sz; + std::atomic<size_t> _sz; std::atomic<uint64_t> _byteSize; vespalib::string _fileName; std::unique_ptr<FastOS_FileInterface> _transLog; diff --git a/slobrok/src/tests/startsome/tstdst.cpp b/slobrok/src/tests/startsome/tstdst.cpp index d83d79b3a00..8a6b3901cc0 100644 --- a/slobrok/src/tests/startsome/tstdst.cpp +++ b/slobrok/src/tests/startsome/tstdst.cpp @@ -176,6 +176,7 @@ TstEnv::MainLoop() FRT_Target *slobrok = getSupervisor()->GetTarget(sbspec.c_str()); slobrok->InvokeAsync(req, 5.0, this); getTransport()->Main(); + getTransport()->WaitFinished(); return 0; } diff --git a/slobrok/src/vespa/slobrok/server/sbenv.cpp b/slobrok/src/vespa/slobrok/server/sbenv.cpp index 0a8b6c60864..9b1b4ce97ec 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.cpp +++ b/slobrok/src/vespa/slobrok/server/sbenv.cpp @@ -183,6 +183,7 @@ SBEnv::MainLoop() LOG(debug, "slobrok: starting main event loop"); EV_STARTED("slobrok"); getTransport()->Main(); + getTransport()->WaitFinished(); LOG(debug, "slobrok: main event loop done"); } catch (vespalib::Exception &e) { LOG(error, "invalid config: %s", e.what()); diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index 7cb66ab447f..60aebf5a535 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -272,8 +272,9 @@ StateManager::notifyStateListeners() } for (auto* listener : _stateListeners) { listener->handleNewState(); - // If one of them actually altered the state again, abort - // sending events, update states and send new one to all. + // If one of them actually altered the state again, abort + // sending events, update states and send new one to all. + std::lock_guard guard(_stateLock); if (_nextNodeState || _nextSystemState) { break; } diff --git a/yolean/abi-spec.json b/yolean/abi-spec.json index 553a8aa61e1..6285cc54118 100644 --- a/yolean/abi-spec.json +++ b/yolean/abi-spec.json @@ -234,36 +234,6 @@ ], "fields": [] }, - "com.yahoo.yolean.concurrent.Memoized$Closer": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void close(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.yolean.concurrent.Memoized": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.util.function.Supplier", - "java.lang.AutoCloseable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.util.function.Supplier, com.yahoo.yolean.concurrent.Memoized$Closer)", - "public static com.yahoo.yolean.concurrent.Memoized of(java.util.function.Supplier)", - "public java.lang.Object get()", - "public void close()" - ], - "fields": [] - }, "com.yahoo.yolean.concurrent.ResourceFactory": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/Memoized.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/Memoized.java deleted file mode 100644 index e8660504a8a..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/Memoized.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.yahoo.yolean.concurrent; - -import java.util.function.Supplier; - -import static java.util.Objects.requireNonNull; - -/** - * Wraps a lazily initialised resource which needs to be shut down. - * The wrapped supplier may not return {@code null}, and should be retryable on failure. - * If it throws, it will be retried if {@link #get} is retried. A supplier that fails to - * clean up partial state on failure may cause a resource leak. - * - * @author jonmv - */ -public class Memoized<T, E extends Exception> implements Supplier<T>, AutoCloseable { - - /** Provides a tighter bound on the thrown exception type. */ - @FunctionalInterface - public interface Closer<T, E extends Exception> { void close(T t) throws E; } - - private final Object monitor = new Object(); - private final Closer<T, E> closer; - private volatile T wrapped; - private Supplier<T> factory; - - public Memoized(Supplier<T> factory, Closer<T, E> closer) { - this.factory = requireNonNull(factory); - this.closer = requireNonNull(closer); - } - - public static <T extends AutoCloseable> Memoized<T, ?> of(Supplier<T> factory) { - return new Memoized<>(factory, AutoCloseable::close); - } - - @Override - public T get() { - // Double-checked locking: try the variable, and if not initialized, try to initialize it. - if (wrapped == null) synchronized (monitor) { - // Ensure the factory is called only once, by clearing it once successfully called. - if (factory != null) wrapped = requireNonNull(factory.get()); - factory = null; - - // If we found the factory, we won the initialization race, and return normally; otherwise - // if wrapped is non-null, we lost the race, wrapped was set by the winner, and we return; otherwise - // we tried to initialise because wrapped was cleared by closing this, and we fail. - if (wrapped == null) throw new IllegalStateException("already closed"); - } - return wrapped; - } - - @Override - public void close() throws E { - // Alter state only when synchronized with calls to get(). - synchronized (monitor) { - // Ensure we only try to close the generated resource once, by clearing it after picking it up here. - T maybe = wrapped; - wrapped = null; - // Clear the factory, to signal this has been closed. - factory = null; - if (maybe != null) closer.close(maybe); - } - } - -}
\ No newline at end of file diff --git a/yolean/src/test/java/com/yahoo/yolean/concurrent/MemoizedTest.java b/yolean/src/test/java/com/yahoo/yolean/concurrent/MemoizedTest.java deleted file mode 100644 index 7f2f49c75f2..00000000000 --- a/yolean/src/test/java/com/yahoo/yolean/concurrent/MemoizedTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.yahoo.yolean.concurrent; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.Phaser; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.fail; - -/** - * @author jonmv - */ -public class MemoizedTest { - - final Phaser phaser = new Phaser(); - final int threads = 128; - - @Test - public void test() throws ExecutionException, InterruptedException { - var lazy = new Memoized<>(new OnceSupplier(), OnceCloseable::close); - phaser.register(); // test thread - phaser.register(); // whoever calls the factory - - Phaser latch = new Phaser(threads + 1); - ExecutorService executor = Executors.newFixedThreadPool(threads); - List<Future<?>> futures = new ArrayList<>(); - for (int i = 0; i < 128; i++) { - futures.add(executor.submit(() -> { - latch.arriveAndAwaitAdvance(); - lazy.get().rendezvous(); - while (true) lazy.get(); - })); - } - - // All threads waiting for latch, will race to factory - latch.arriveAndAwaitAdvance(); - - // One thread waiting in factory, the others are blocked, will go to rendezvous - phaser.arriveAndAwaitAdvance(); - - // All threads waiting in rendezvous, will repeatedly get until failure - phaser.arriveAndAwaitAdvance(); - - // Unsynchronized close should be detected by all threads - lazy.close(); - - // Close should carry through only once - lazy.close(); - - assertEquals("already closed", - assertThrows(IllegalStateException.class, lazy::get).getMessage()); - - for (Future<?> future : futures) - assertEquals("java.lang.IllegalStateException: already closed", - assertThrows(ExecutionException.class, future::get).getMessage()); - - executor.shutdown(); - } - - @Test - public void closeBeforeFirstGet() throws Exception { - OnceSupplier supplier = new OnceSupplier(); - Memoized<OnceCloseable, ?> lazy = Memoized.of(supplier); - lazy.close(); - assertEquals("already closed", - assertThrows(IllegalStateException.class, lazy::get).getMessage()); - lazy.close(); - assertFalse(supplier.initialized.get()); - } - - class OnceSupplier implements Supplier<OnceCloseable> { - final AtomicBoolean initialized = new AtomicBoolean(); - @Override public OnceCloseable get() { - phaser.arriveAndAwaitAdvance(); - if ( ! initialized.compareAndSet(false, true)) fail("initialized more than once"); - phaser.bulkRegister(threads - 1); // register all the threads who didn't get the factory - return new OnceCloseable(); - } - } - - class OnceCloseable implements AutoCloseable { - final AtomicBoolean closed = new AtomicBoolean(); - @Override public void close() { - if ( ! closed.compareAndSet(false, true)) fail("closed more than once"); - } - void rendezvous() { - phaser.arriveAndAwaitAdvance(); - } - } - -} |