diff options
Diffstat (limited to 'config-model/src/main')
40 files changed, 743 insertions, 278 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 16affbd7b0e..c8f088509c5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.ProvisionLogger; import java.util.ArrayList; @@ -234,7 +235,7 @@ public class InMemoryProvisioner implements HostProvisioner { // Minimal capacity policies private NodeResources decideResources(NodeResources resources) { if (defaultNodeResources.isUnspecified()) return resources; - return resources.withUnspecifiedNumbersFrom(defaultNodeResources); + return resources.withUnspecifiedFieldsFrom(defaultNodeResources); } private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResourcesOrUnspecified, diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 6007a1cf4b1..1ff85c9c89f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -22,6 +22,7 @@ import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -30,6 +31,7 @@ import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -109,7 +111,7 @@ public class RankProfile implements Cloneable { private String inheritedSummaryFeaturesProfileName; private Set<ReferenceNode> matchFeatures; - private Set<String> hiddenMatchFeatures; + private Set<ReferenceNode> hiddenMatchFeatures; private String inheritedMatchFeaturesProfileName; private Set<ReferenceNode> rankFeatures; @@ -607,7 +609,7 @@ public class RankProfile implements Cloneable { .orElse(Set.of()); } - public Set<String> getHiddenMatchFeatures() { + public Set<ReferenceNode> getHiddenMatchFeatures() { if (hiddenMatchFeatures != null) return Collections.unmodifiableSet(hiddenMatchFeatures); return uniquelyInherited(p -> p.getHiddenMatchFeatures(), f -> ! f.isEmpty(), "hidden match features") .orElse(Set.of()); @@ -626,15 +628,13 @@ public class RankProfile implements Cloneable { } private void addImplicitMatchFeatures(List<FeatureList> list) { - if (matchFeatures == null) - matchFeatures = new LinkedHashSet<>(); if (hiddenMatchFeatures == null) hiddenMatchFeatures = new LinkedHashSet<>(); + var current = getMatchFeatures(); for (var features : list) { for (ReferenceNode feature : features) { - if (! matchFeatures.contains(feature)) { - matchFeatures.add(feature); - hiddenMatchFeatures.add(feature.toString()); + if (! current.contains(feature)) { + hiddenMatchFeatures.add(feature); } } } @@ -1058,21 +1058,45 @@ public class RankProfile implements Cloneable { functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); allFunctionsCached = null; + var context = new RankProfileTransformContext(this, + queryProfiles, + featureTypes, + importedModels, + constants(), + inlineFunctions); + var allNormalizers = getFeatureNormalizers(); + verifyNoNormalizers("first-phase expression", firstPhaseRanking, allNormalizers, context); + verifyNoNormalizers("second-phase expression", secondPhaseRanking, allNormalizers, context); + for (ReferenceNode mf : getMatchFeatures()) { + verifyNoNormalizers("match-feature " + mf, mf, allNormalizers, context); + } + for (ReferenceNode sf : getSummaryFeatures()) { + verifyNoNormalizers("summary-feature " + sf, sf, allNormalizers, context); + } if (globalPhaseRanking != null) { - var context = new RankProfileTransformContext(this, - queryProfiles, - featureTypes, - importedModels, - constants(), - inlineFunctions); var needInputs = new HashSet<String>(); + Set<String> userDeclaredMatchFeatures = new HashSet<>(); + for (ReferenceNode mf : getMatchFeatures()) { + userDeclaredMatchFeatures.add(mf.toString()); + } var recorder = new InputRecorder(needInputs); - if (matchFeatures != null) { - for (ReferenceNode mf : matchFeatures) { - recorder.alreadyHandled(mf.toString()); + recorder.alreadyMatchFeatures(userDeclaredMatchFeatures); + recorder.addKnownNormalizers(allNormalizers.keySet()); + recorder.process(globalPhaseRanking.function().getBody(), context); + for (var normalizerName : recorder.normalizersUsed()) { + var normalizer = allNormalizers.get(normalizerName); + var func = functions.get(normalizer.input()); + if (func != null) { + verifyNoNormalizers("normalizer input " + normalizer.input(), func, allNormalizers, context); + if (! userDeclaredMatchFeatures.contains(normalizer.input())) { + var subRecorder = new InputRecorder(needInputs); + subRecorder.alreadyMatchFeatures(userDeclaredMatchFeatures); + subRecorder.process(func.function().getBody(), context); + } + } else { + needInputs.add(normalizer.input()); } } - recorder.process(globalPhaseRanking.function().getBody(), context); List<FeatureList> addIfMissing = new ArrayList<>(); for (String input : needInputs) { if (input.startsWith("constant(") || input.startsWith("query(")) { @@ -1630,4 +1654,70 @@ public class RankProfile implements Cloneable { } + public static record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { + @Override + public String toString() { + return "normalizer{name=" + name + ",input=" + input + ",algo=" + algo + ",k=" + kparam + "}"; + } + private static long hash(String s) { + int bob = com.yahoo.collections.BobHash.hash(s); + return bob + 0x100000000L; + } + public static RankFeatureNormalizer linear(Reference original, Reference inputRef) { + long h = hash(original.toString()); + String name = "normalize@" + h + "@linear"; + return new RankFeatureNormalizer(original, name, inputRef.toString(), "LINEAR", 0.0); + } + public static RankFeatureNormalizer rrank(Reference original, Reference inputRef, double k) { + long h = hash(original.toString()); + String name = "normalize@" + h + "@rrank"; + return new RankFeatureNormalizer(original, name, inputRef.toString(), "RRANK", k); + } + } + + private List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); + + public Map<String, RankFeatureNormalizer> getFeatureNormalizers() { + Map<String, RankFeatureNormalizer> all = new LinkedHashMap<>(); + for (var inheritedProfile : inherited()) { + all.putAll(inheritedProfile.getFeatureNormalizers()); + } + for (var n : featureNormalizers) { + all.put(n.name(), n); + } + return all; + } + + public void addFeatureNormalizer(RankFeatureNormalizer n) { + if (functions.get(n.name()) != null) { + throw new IllegalArgumentException("cannot use name '" + name + "' for both function and normalizer"); + } + featureNormalizers.add(n); + } + + private void verifyNoNormalizers(String where, RankingExpressionFunction f, Map<String, RankFeatureNormalizer> allNormalizers, RankProfileTransformContext context) { + if (f == null) return; + verifyNoNormalizers(where, f.function(), allNormalizers, context); + } + + private void verifyNoNormalizers(String where, ExpressionFunction func, Map<String, RankFeatureNormalizer> allNormalizers, RankProfileTransformContext context) { + if (func == null) return; + var body = func.getBody(); + if (body == null) return; + verifyNoNormalizers(where, body.getRoot(), allNormalizers, context); + } + + private void verifyNoNormalizers(String where, ExpressionNode node, Map<String, RankFeatureNormalizer> allNormalizers, RankProfileTransformContext context) { + var needInputs = new HashSet<String>(); + var recorder = new InputRecorder(needInputs); + recorder.process(node, context); + for (var input : needInputs) { + var normalizer = allNormalizers.get(input); + if (normalizer != null) { + throw new IllegalArgumentException("Cannot use " + normalizer.original() + " from " + where + ", only valid in global-phase expression"); + } + } + } + + } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 05e5f17ea3d..68fa0fe6de9 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -54,6 +54,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final String name; private final Compressor.Compression compressedProperties; + private final Map<String, RankProfile.RankFeatureNormalizer> featureNormalizers; /** The compiled profile this is created from. */ private final Collection<RankProfile.Constant> constants; @@ -66,13 +67,14 @@ public class RawRankProfile implements RankProfilesConfig.Producer { this.name = rankProfile.name(); /* * Forget the RankProfiles as soon as possible. They can become very large and memory hungry - * Especially do not refer then through any member variables due to the RawRankProfile living forever. + * Especially do not refer them through any member variables due to the RawRankProfile living forever. */ RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); constants = compiled.constants().values(); onnxModels = compiled.onnxModels().values(); - compressedProperties = compress(new Deriver(compiled, attributeFields, deployProperties, queryProfiles) - .derive(largeExpressions)); + var deriver = new Deriver(compiled, attributeFields, deployProperties, queryProfiles); + compressedProperties = compress(deriver.derive(largeExpressions)); + this.featureNormalizers = compiled.getFeatureNormalizers(); } public Collection<RankProfile.Constant> constants() { return constants; } @@ -111,6 +113,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer { b.fef(fefB); } + private void buildNormalizers(RankProfilesConfig.Rankprofile.Builder b) { + for (var normalizer : featureNormalizers.values()) { + var nBuilder = new RankProfilesConfig.Rankprofile.Normalizer.Builder(); + nBuilder.name(normalizer.name()); + nBuilder.input(normalizer.input()); + var algo = RankProfilesConfig.Rankprofile.Normalizer.Algo.Enum.valueOf(normalizer.algo()); + nBuilder.algo(algo); + nBuilder.kparam(normalizer.kparam()); + b.normalizer(nBuilder); + } + } + /** * Returns the properties of this as an unmodifiable list. * Note: This method is expensive. @@ -121,6 +135,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public void getConfig(RankProfilesConfig.Builder builder) { RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName()); getRankProperties(b); + buildNormalizers(b); builder.rankprofile(b); } @@ -134,7 +149,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); private final Set<ReferenceNode> summaryFeatures; private final Set<ReferenceNode> matchFeatures; - private final Collection<String> hiddenMatchFeatures; + private final Set<ReferenceNode> hiddenMatchFeatures; private final Set<ReferenceNode> rankFeatures; private final Map<String, String> featureRenames = new java.util.LinkedHashMap<>(); private final List<RankProfile.RankProperty> rankProperties; @@ -188,6 +203,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); matchFeatures = new LinkedHashSet<>(compiled.getMatchFeatures()); hiddenMatchFeatures = compiled.getHiddenMatchFeatures(); + matchFeatures.addAll(hiddenMatchFeatures); rankFeatures = compiled.getRankFeatures(); rerankCount = compiled.getRerankCount(); globalPhaseRerankCount = compiled.getGlobalPhaseRerankCount(); @@ -414,8 +430,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (ReferenceNode feature : matchFeatures) { properties.add(new Pair<>("vespa.match.feature", feature.toString())); } - for (String feature : hiddenMatchFeatures) { - properties.add(new Pair<>("vespa.hidden.matchfeature", feature)); + for (ReferenceNode feature : hiddenMatchFeatures) { + properties.add(new Pair<>("vespa.hidden.matchfeature", feature.toString())); } for (ReferenceNode feature : rankFeatures) { properties.add(new Pair<>("vespa.dump.feature", feature.toString())); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index ddb6b004070..300a55e521a 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -155,7 +155,8 @@ public class SummaryClass extends Derived { summaryField.getTransform() == SummaryTransform.GEOPOS || summaryField.getTransform() == SummaryTransform.POSITIONS || summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || - summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) + summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER || + summaryField.getTransform() == SummaryTransform.TOKENS) { return summaryField.getSingleSource(); } else if (summaryField.getTransform().isDynamic()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java index c1e6dd2aea3..2f60cd8eb06 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -92,6 +92,8 @@ public class SummaryClassField { return Type.FEATUREDATA; } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) { return Type.FEATUREDATA; + } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) { + return Type.JSONSTRING; } else { return Type.LONGSTRING; } diff --git a/config-model/src/main/java/com/yahoo/schema/document/Ranking.java b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java index d00abfcb9aa..2a2b1431057 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Ranking.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java @@ -44,9 +44,8 @@ public class Ranking implements Cloneable, Serializable { /** Returns true if the given rank settings are the same */ @Override public boolean equals(Object o) { - if ( ! (o instanceof Ranking)) return false; + if ( ! (o instanceof Ranking other)) return false; - Ranking other=(Ranking)o; if (this.filter != other.filter) return false; if (this.literal != other.literal) return false; if (this.normal != other.normal) return false; diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java index cf46bedf223..42c8147b3dc 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java @@ -35,7 +35,8 @@ public class ExpressionTransforms { new FunctionShadower(), new TensorMaxMinTransformer(), new Simplifier(), - new BooleanExpressionTransformer()); + new BooleanExpressionTransformer(), + new NormalizerFunctionExpander()); } public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index 1128aaf3681..ab18f9c83db 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -14,6 +14,7 @@ import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; import com.yahoo.tensor.functions.Generate; import java.io.StringReader; +import java.util.Collection; import java.util.HashSet; import java.util.Set; import java.util.logging.Logger; @@ -29,19 +30,35 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { private final Set<String> neededInputs; private final Set<String> handled = new HashSet<>(); + private final Set<String> availableNormalizers = new HashSet<>(); + private final Set<String> usedNormalizers = new HashSet<>(); public InputRecorder(Set<String> target) { this.neededInputs = target; } public void process(RankingExpression expression, RankProfileTransformContext context) { - transform(expression.getRoot(), new InputRecorderContext(context)); + process(expression.getRoot(), context); } - public void alreadyHandled(String name) { - handled.add(name); + public void process(ExpressionNode node, RankProfileTransformContext context) { + transform(node, new InputRecorderContext(context)); } + public void alreadyMatchFeatures(Collection<String> matchFeatures) { + for (String mf : matchFeatures) { + handled.add(mf); + } + } + + public void addKnownNormalizers(Collection<String> names) { + for (String name : names) { + availableNormalizers.add(name); + } + } + + public Set<String> normalizersUsed() { return this.usedNormalizers; } + @Override public ExpressionNode transform(ExpressionNode node, InputRecorderContext context) { if (node instanceof ReferenceNode r) { @@ -77,6 +94,10 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { if (simpleFunctionOrIdentifier && context.localVariables().contains(name)) { return; } + if (simpleFunctionOrIdentifier && availableNormalizers.contains(name)) { + usedNormalizers.add(name); + return; + } if (ref.isSimpleRankingExpressionWrapper()) { name = ref.simpleArgument().get(); simpleFunctionOrIdentifier = true; @@ -113,13 +134,21 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { } } if ("onnx".equals(name)) { - if (args.size() != 1) { + if (args.size() < 1) { throw new IllegalArgumentException("expected name of ONNX model as argument: " + feature); } var arg = args.expressions().get(0); var models = context.rankProfile().onnxModels(); var model = models.get(arg.toString()); if (model == null) { + var tmp = OnnxModelTransformer.transformFeature(feature, context.rankProfile()); + if (tmp instanceof ReferenceNode newRefNode) { + args = newRefNode.getArguments(); + arg = args.expressions().get(0); + model = models.get(arg.toString()); + } + } + if (model == null) { throw new IllegalArgumentException("missing onnx model: " + arg); } model.getInputMap().forEach((__, onnxInput) -> { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java new file mode 100644 index 00000000000..a8fee966656 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java @@ -0,0 +1,134 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.RankProfile.RankFeatureNormalizer; +import com.yahoo.searchlib.rankingexpression.evaluation.BooleanValue; +import com.yahoo.searchlib.rankingexpression.rule.OperationNode; +import com.yahoo.searchlib.rankingexpression.rule.Operator; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.IfNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.searchlib.rankingexpression.transform.TransformContext; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.functions.Generate; + +import java.io.StringReader; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.ArrayList; + +/** + * Recognizes pseudo-functions and creates global-phase normalizers + * @author arnej + */ +public class NormalizerFunctionExpander extends ExpressionTransformer<RankProfileTransformContext> { + + public final static String NORMALIZE_LINEAR = "normalize_linear"; + public final static String RECIPROCAL_RANK = "reciprocal_rank"; + public final static String RECIPROCAL_RANK_FUSION = "reciprocal_rank_fusion"; + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode r) { + node = transformReference(r, context); + } + if (node instanceof CompositeNode composite) { + node = transformChildren(composite, context); + } + return node; + } + + private ExpressionNode transformReference(ReferenceNode node, RankProfileTransformContext context) { + Reference ref = node.reference(); + String name = ref.name(); + if (ref.output() != null) { + return node; + } + var f = context.rankProfile().getFunctions().get(name); + if (f != null) { + // never transform declared functions + return node; + } + return switch(name) { + case RECIPROCAL_RANK_FUSION -> transform(expandRRF(ref), context); + case NORMALIZE_LINEAR -> transformNormLin(ref, context); + case RECIPROCAL_RANK -> transformRRank(ref, context); + default -> node; + }; + } + + private ExpressionNode expandRRF(Reference ref) { + var args = ref.arguments(); + if (args.size() < 2) { + throw new IllegalArgumentException("must have at least 2 arguments: " + ref); + } + List<ExpressionNode> children = new ArrayList<>(); + List<Operator> operators = new ArrayList<>(); + for (var arg : args.expressions()) { + if (! children.isEmpty()) operators.add(Operator.plus); + children.add(new ReferenceNode(RECIPROCAL_RANK, List.of(arg), null)); + } + // must be further transformed (see above) + return new OperationNode(children, operators); + } + + private ExpressionNode transformNormLin(Reference ref, RankProfileTransformContext context) { + var args = ref.arguments(); + if (args.size() != 1) { + throw new IllegalArgumentException("must have exactly 1 argument: " + ref); + } + var input = args.expressions().get(0); + if (input instanceof ReferenceNode inputRefNode) { + var inputRef = inputRefNode.reference(); + RankFeatureNormalizer normalizer = RankFeatureNormalizer.linear(ref, inputRef); + context.rankProfile().addFeatureNormalizer(normalizer); + var newRef = Reference.fromIdentifier(normalizer.name()); + return new ReferenceNode(newRef); + } else { + throw new IllegalArgumentException("the first argument must be a simple feature: " + ref + " => " + input.getClass()); + } + } + + private ExpressionNode transformRRank(Reference ref, RankProfileTransformContext context) { + var args = ref.arguments(); + if (args.size() < 1 || args.size() > 2) { + throw new IllegalArgumentException("must have 1 or 2 arguments: " + ref); + } + double k = 60.0; + if (args.size() == 2) { + var kArg = args.expressions().get(1); + if (kArg instanceof ConstantNode kNode) { + k = kNode.getValue().asDouble(); + } else { + throw new IllegalArgumentException("the second argument (k) must be a constant in: " + ref); + } + } + var input = args.expressions().get(0); + if (input instanceof ReferenceNode inputRefNode) { + var inputRef = inputRefNode.reference(); + RankFeatureNormalizer normalizer = RankFeatureNormalizer.rrank(ref, inputRef, k); + context.rankProfile().addFeatureNormalizer(normalizer); + var newRef = Reference.fromIdentifier(normalizer.name()); + return new ReferenceNode(newRef); + } else { + throw new IllegalArgumentException("the first argument must be a simple feature: " + ref); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 7c6d62580cb..bc0e16abbe3 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import java.util.Locale; import java.util.Map; +import java.util.logging.Level; /** * Helper for converting ParsedField etc to SDField with settings @@ -137,7 +138,7 @@ public class ConvertParsedFields { } // from grammar, things that can be inside struct-field block - private void convertCommonFieldSettings(SDField field, ParsedField parsed) { + private void convertCommonFieldSettings(Schema schema, SDField field, ParsedField parsed) { convertMatchSettings(field, parsed.matchSettings()); var indexing = parsed.getIndexing(); if (indexing.isPresent()) { @@ -152,7 +153,12 @@ public class ConvertParsedFields { for (var summaryField : parsed.getSummaryFields()) { var dataType = field.getDataType(); var otherType = summaryField.getType(); - if (otherType != null) { + if (otherType != null && summaryField.getHasExplicitType()) { + schema.getDeployLogger().log(Level.FINE, () -> "For " + schema.getName() + + ", field '" + field.getName() + + "', summary '" + summaryField.name() + + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + + " Remove the type specification to silence this warning."); dataType = context.resolveType(otherType); } convertSummaryField(field, summaryField, dataType); @@ -161,7 +167,7 @@ public class ConvertParsedFields { field.addQueryCommand(command); } for (var structField : parsed.getStructFields()) { - convertStructField(field, structField); + convertStructField(schema, field, structField); } if (parsed.hasLiteral()) { field.getRanking().setLiteral(true); @@ -174,13 +180,13 @@ public class ConvertParsedFields { } } - private void convertStructField(SDField field, ParsedField parsed) { + private void convertStructField(Schema schema, SDField field, ParsedField parsed) { SDField structField = field.getStructField(parsed.name()); if (structField == null ) { throw new IllegalArgumentException("Struct field '" + parsed.name() + "' has not been defined in struct " + "for field '" + field.getName() + "'."); } - convertCommonFieldSettings(structField, parsed); + convertCommonFieldSettings(schema, structField, parsed); } private void convertExtraFieldSettings(SDField field, ParsedField parsed) { @@ -217,6 +223,8 @@ public class ConvertParsedFields { transform = SummaryTransform.MATCHED_ELEMENTS_FILTER; } else if (parsed.getDynamic()) { transform = SummaryTransform.DYNAMICTEASER; + } else if (parsed.getTokens()) { + transform = SummaryTransform.TOKENS; } if (parsed.getBolded()) { transform = transform.bold(); @@ -278,7 +286,7 @@ public class ConvertParsedFields { String name = parsed.name(); DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(document, name, dataType); - convertCommonFieldSettings(field, parsed); + convertCommonFieldSettings(schema, field, parsed); convertExtraFieldSettings(field, parsed); document.addField(field); return field; @@ -288,7 +296,7 @@ public class ConvertParsedFields { String name = parsed.name(); DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(schema.getDocument(), name, dataType); - convertCommonFieldSettings(field, parsed); + convertCommonFieldSettings(schema, field, parsed); convertExtraFieldSettings(field, parsed); schema.addExtraField(field); } @@ -305,7 +313,7 @@ public class ConvertParsedFields { for (var parsedField : parsed.getFields()) { var fieldType = context.resolveType(parsedField.getType()); var field = new SDField(document, parsedField.name(), fieldType); - convertCommonFieldSettings(field, parsedField); + convertCommonFieldSettings(schema, field, parsedField); structProxy.addField(field); if (parsedField.hasIdOverride()) { structProxy.setFieldId(field, parsedField.idOverride()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 9145934501c..7e19cb4a0ae 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -11,11 +11,13 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.PositionDataType; import com.yahoo.schema.DefaultRankProfile; import com.yahoo.schema.DocumentOnlySchema; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.UnrankedRankProfile; +import com.yahoo.schema.derived.SummaryClass; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.TemporaryImportedField; @@ -25,7 +27,9 @@ import com.yahoo.vespa.documentmodel.SummaryField; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.logging.Level; import java.util.Map; import java.util.Optional; @@ -137,7 +141,86 @@ public class ConvertParsedSchemas { schema.addDocument(document); } - private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { + /* + * Helper class for resolving data type for a document summary. Summary type is still + * used internally in config model when generating and processing indexing scripts. + * See DynamicSummaryTransformUtils class comment for more details. + * + * This kind of resolving is a temporary measure until the use of summary fields have + * been eliminated from indexing scripts and are no longer used to extend the document + * type. At that time, the data type of a summary field is no longer relevant. + */ + private class SummaryFieldTypeResolver { + + private final Schema schema; + private final Map<String, ParsedSummaryField> summaryFields = new LinkedHashMap<String, ParsedSummaryField>(); + private static final String zCurveSuffix = new String("_zcurve"); + + public SummaryFieldTypeResolver(Schema schema, List<ParsedDocumentSummary> parsed) { + this.schema = schema; + for (var docsum : parsed) { + for (var field : docsum.getSummaryFields()) { + summaryFields.put(field.name(), field); + } + } + } + + private boolean isPositionAttribute(Schema schema, String sourceFieldName) { + if (!sourceFieldName.endsWith(zCurveSuffix)) { + return false; + } + var name = sourceFieldName.substring(0, sourceFieldName.length() - zCurveSuffix.length()); + var field = schema.getField(name); + return (field.getDataType().equals(PositionDataType.INSTANCE)); + } + + + private String getSingleSource(ParsedSummaryField parsedField) { + if (parsedField.getSources().size() == 1) { + return parsedField.getSources().get(0); + } + return parsedField.name(); + } + + public DataType resolve(ParsedDocumentSummary docsum, ParsedSummaryField parsedField) { + var seen = new LinkedHashSet<String>(); + var origName = parsedField.name(); + while (true) { + if (seen.contains(parsedField.name())) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "' summary class '" + docsum.name() + + "' summary field '" + origName + + "': Source loop detected for summary field '" + parsedField.name() + "'"); + } + seen.add(parsedField.name()); + if (parsedField.getSources().size() >= 2) { + return DataType.STRING; // Flattening, streaming search + } + var source = getSingleSource(parsedField); + if (source.equals(SummaryClass.DOCUMENT_ID_FIELD)) { + return DataType.STRING; // Reserved source field name + } else if (isPositionAttribute(schema, source)) { + return DataType.LONG; // Extra field with suffix is added later for positions + } + var field = schema.getField(source); + if (field != null) { + return field.getDataType(); + } else if (schema.temporaryImportedFields().isPresent() && + schema.temporaryImportedFields().get().hasField(source)) { + return null; // Imported field, cannot resolve now + } else if (source.equals(parsedField.name()) || !summaryFields.containsKey(source)) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', summary class '" + docsum.name() + + "', summary field '" + parsedField.name() + + "': there is no valid source '" + source + "'."); + } + parsedField = summaryFields.get(source); + } + } + } + + private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext, + SummaryFieldTypeResolver sfResolver) { var docsum = new DocumentSummary(parsed.name(), schema); parsed.getInherited().forEach(inherited -> docsum.addInherited(inherited)); if (parsed.getFromDisk()) { @@ -148,10 +231,17 @@ public class ConvertParsedSchemas { } for (var parsedField : parsed.getSummaryFields()) { var parsedType = parsedField.getType(); + if (parsedType != null) { + var log = schema.getDeployLogger(); + log.log(Level.FINE, () -> "For " + schema.getName() + + ", document-summary '" + parsed.name() + + "', summary field '" + parsedField.name() + + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + + " Remove the type specification to silence this warning."); + } DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null; - var existingField = schema.getField(parsedField.name()); - if (existingField != null) { - var existingType = existingField.getDataType(); + DataType existingType = sfResolver.resolve(parsed, parsedField); + if (existingType != null) { if (dataType == null) { dataType = existingType; } else if (!dataType.equals(existingType)) { @@ -161,10 +251,9 @@ public class ConvertParsedSchemas { } } } - if (dataType == null) { - throw new IllegalArgumentException("Missing data-type for summary field " + parsedField.name() + " in document-summary " + parsed.name()); - } - var summaryField = new SummaryField(parsedField.name(), dataType); + var summaryField = (dataType == null) ? + SummaryField.createWithUnresolvedType(parsedField.name()) : + new SummaryField(parsedField.name(), dataType); // XXX does not belong here: summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); @@ -206,6 +295,7 @@ public class ConvertParsedSchemas { } parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); var typeContext = typeConverter.makeContext(parsed.getDocument()); + var sfResolver = new SummaryFieldTypeResolver(schema, parsed.getDocumentSummaries()); var fieldConverter = new ConvertParsedFields(typeContext, convertedStructs); convertDocument(schema, parsed.getDocument(), fieldConverter); for (var field : parsed.getFields()) { @@ -214,12 +304,12 @@ public class ConvertParsedSchemas { for (var index : parsed.getIndexes()) { fieldConverter.convertExtraIndex(schema, index); } - for (var docsum : parsed.getDocumentSummaries()) { - convertDocumentSummary(schema, docsum, typeContext); - } for (var importedField : parsed.getImportedFields()) { convertImportField(schema, importedField); } + for (var docsum : parsed.getDocumentSummaries()) { + convertDocumentSummary(schema, docsum, typeContext, sfResolver); + } for (var fieldSet : parsed.getFieldSets()) { convertFieldSet(schema, fieldSet); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java index 1d5d73635e7..8f9733d2595 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java @@ -18,6 +18,8 @@ class ParsedSummaryField extends ParsedBlock { private boolean isMEO = false; private boolean isFull = false; private boolean isBold = false; + private boolean isTokens = false; + private boolean hasExplicitType = false; private final List<String> sources = new ArrayList<>(); private final List<String> destinations = new ArrayList<>(); @@ -37,6 +39,8 @@ class ParsedSummaryField extends ParsedBlock { boolean getDynamic() { return isDyn; } boolean getFull() { return isFull; } boolean getMatchedElementsOnly() { return isMEO; } + boolean getTokens() { return isTokens; } + boolean getHasExplicitType() { return hasExplicitType; } void addDestination(String dst) { destinations.add(dst); } void addSource(String src) { sources.add(src); } @@ -44,6 +48,8 @@ class ParsedSummaryField extends ParsedBlock { void setDynamic() { this.isDyn = true; } void setFull() { this.isFull = true; } void setMatchedElementsOnly() { this.isMEO = true; } + void setTokens() { this.isTokens = true; } + void setHasExplicitType() { this.hasExplicitType = true; } void setType(ParsedType value) { verifyThat(type == null, "Cannot change type from ", type, "to", value); this.type = value; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java index 5b72381bfb1..762279e3871 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java @@ -2,6 +2,8 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.document.ImmutableImportedComplexSDField; @@ -13,17 +15,17 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.stream.Stream; /** - * Adds the attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported + * Adds the data type and attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported * field as source. * * @author bjorncs */ -public class AddAttributeTransformToSummaryOfImportedFields extends Processor { +public class AddDataTypeAndTransformToSummaryOfImportedFields extends Processor { - public AddAttributeTransformToSummaryOfImportedFields(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { + public AddDataTypeAndTransformToSummaryOfImportedFields(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } @@ -39,19 +41,29 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { private void setTransform(ImmutableSDField field) { if (field instanceof ImmutableImportedComplexSDField) { - getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform); + getSummaryFieldsForImportedField(field).forEach(summaryField -> setAttributeCombinerTransform(field, summaryField)); } else { - getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); + getSummaryFieldsForImportedField(field).forEach(summaryField -> setAttributeTransform(field, summaryField)); } } - private static void setAttributeTransform(SummaryField summaryField) { + private static void setAttributeTransform(ImmutableSDField field, SummaryField summaryField) { + if (summaryField.hasUnresolvedType()) { + if (field.getDataType().equals(DataType.LONG) && summaryField.getTransform().equals(SummaryTransform.GEOPOS)) { + summaryField.setResolvedDataType(PositionDataType.INSTANCE); + } else { + summaryField.setResolvedDataType(field.getDataType()); + } + } if (summaryField.getTransform() == SummaryTransform.NONE) { summaryField.setTransform(SummaryTransform.ATTRIBUTE); } } - private static void setAttributeCombinerTransform(SummaryField summaryField) { + private static void setAttributeCombinerTransform(ImmutableSDField field, SummaryField summaryField) { + if (summaryField.hasUnresolvedType()) { + summaryField.setResolvedDataType(field.getDataType()); + } if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) { // This field already has the correct transform. return; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java new file mode 100644 index 00000000000..dd6f118d113 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java @@ -0,0 +1,82 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + +/** + * Adds the corresponding summary transform for all "documentid" summary fields. + * For summary fields without an existing transform: + * - Adds the attribute transforms where the source field has an attribute vector. + * - Adds the attribute combiner transform where the source field is a struct field where all subfields have attribute + * vector. + * - Add the copy transform where the source field is a struct or map field with a different name. + * + * @author geirst + */ +public class AdjustSummaryTransforms extends Processor { + + public AdjustSummaryTransforms(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var summary : schema.getSummaries().values()) { + for (var summaryField : summary.getSummaryFields().values()) { + makeDocumentIdTransformIfAppropriate(summaryField); + makeAttributeTransformIfAppropriate(summaryField, schema); + makeAttributeCombinerTransformIfAppropriate(summaryField, schema); + makeCopyTransformIfAppropriate(summaryField, schema); + } + } + } + + private void makeDocumentIdTransformIfAppropriate(SummaryField summaryField) + { + if (summaryField.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) { + summaryField.setTransform(SummaryTransform.DOCUMENT_ID); + } + } + + /** If the source is an attribute, make this use the attribute transform */ + private void makeAttributeTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() != SummaryTransform.NONE) return; + Attribute attribute = schema.getAttribute(summaryField.getSingleSource()); + if (attribute == null) return; + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + + /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ + private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); + if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { + summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } + } + } + + /* + * This function must be called after makeAttributeCombinerTransformIfAppropriate(). + */ + private void makeCopyTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); + if (source != null && source.usesStructOrMap() && summaryField.hasExplicitSingleSource()) { + summaryField.setTransform(SummaryTransform.COPY); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index 1d279242895..e4116c3f9d5 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -78,7 +78,8 @@ public class IndexingOutputs extends Processor { return; } dynamicSummary.add(summaryName); - } else if (summaryTransform != SummaryTransform.ATTRIBUTE) { + } else if (summaryTransform != SummaryTransform.ATTRIBUTE && + summaryTransform != SummaryTransform.TOKENS) { staticSummary.add(summaryName); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java index 610021c510d..420df3ee575 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java @@ -41,6 +41,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue; if (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER) continue; if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) continue; + if (summaryField.getTransform() == SummaryTransform.TOKENS) continue; defaultSummary.add(summaryField.clone()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index 89e6a1533d0..c23d87e9eba 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -49,15 +49,16 @@ public class Processing { DictionaryProcessor::new, WordMatch::new, ImportedFieldsResolver::new, + AddDataTypeAndTransformToSummaryOfImportedFields::new, ImplicitSummaries::new, ImplicitSummaryFields::new, AdjustPositionSummaryFields::new, - SummaryTransformForDocumentId::new, SummaryConsistency::new, + AdjustSummaryTransforms::new, SummaryNamesFieldCollisions::new, SummaryFieldsMustHaveValidSource::new, + TokensTransformValidator::new, MatchedElementsOnlyResolver::new, - AddAttributeTransformToSummaryOfImportedFields::new, MakeDefaultSummaryTheSuperSet::new, Bolding::new, AttributeProperties::new, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java index f4d2faf9444..0987521831b 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java @@ -9,6 +9,7 @@ import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConst import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Arrays; +import java.util.HashSet; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -46,8 +47,28 @@ public class ReservedFunctionNames extends Processor { } private static Set<String> getReservedNames() { - return Arrays.stream(RankingExpressionParserConstants.tokenImage) - .map(token -> token.substring(1, token.length()-1)).collect(Collectors.toUnmodifiableSet()); + Set<String> temp = new HashSet<>(); + Arrays.stream(RankingExpressionParserConstants.tokenImage) + .map(token -> token.substring(1, token.length()-1)).forEach(name -> temp.add(name)); + temp.add("attribute"); + temp.add("constant"); + temp.add("customTokenInputIds"); + temp.add("firstphase"); + temp.add("globalphase"); + temp.add("normalize_linear"); + temp.add("onnx"); + temp.add("onnx_vespa"); + temp.add("query"); + temp.add("reciprocal_rank"); + temp.add("reciprocal_rank_fusion"); + temp.add("secondphase"); + temp.add("tensor"); + temp.add("tokenAttentionMask"); + temp.add("tokenInputIds"); + temp.add("tokenTypeIds"); + temp.add("value"); + temp.add("xgboost"); + return Set.copyOf(temp); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java index a7ff50ffcca..4b214e00d65 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java @@ -8,14 +8,11 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; - /** * Ensure that summary field transforms for fields having the same name * are consistent across summary classes @@ -35,9 +32,6 @@ public class SummaryConsistency extends Processor { for (SummaryField summaryField : summary.getSummaryFields().values()) { assertConsistency(summaryField, schema, validate); - makeAttributeTransformIfAppropriate(summaryField, schema); - makeAttributeCombinerTransformIfAppropriate(summaryField, schema); - makeCopyTransformIfAppropriate(summaryField, schema); } } } @@ -60,38 +54,6 @@ public class SummaryConsistency extends Processor { } } - /** If the source is an attribute, make this use the attribute transform */ - private void makeAttributeTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() != SummaryTransform.NONE) return; - Attribute attribute = schema.getAttribute(summaryField.getSingleSource()); - if (attribute == null) return; - summaryField.setTransform(SummaryTransform.ATTRIBUTE); - } - - /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ - private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() == SummaryTransform.NONE) { - String sourceFieldName = summaryField.getSingleSource(); - ImmutableSDField source = schema.getField(sourceFieldName); - if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { - summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); - } - } - } - - /* - * This function must be called after makeAttributeCombinerTransformIfAppropriate(). - */ - private void makeCopyTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() == SummaryTransform.NONE) { - String sourceFieldName = summaryField.getSingleSource(); - ImmutableSDField source = schema.getField(sourceFieldName); - if (source != null && source.usesStructOrMap() && summaryField.hasExplicitSingleSource()) { - summaryField.setTransform(SummaryTransform.COPY); - } - } - } - private void assertConsistentTypes(SummaryField existing, SummaryField seen) { if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java deleted file mode 100644 index 388aa93e81c..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.schema.RankProfileRegistry; -import com.yahoo.schema.Schema; -import com.yahoo.schema.derived.SummaryClass; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Adds the corresponding summary transform for all "documentid" summary fields. - * - * @author geirst - */ -public class SummaryTransformForDocumentId extends Processor { - - public SummaryTransformForDocumentId(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (var summary : schema.getSummaries().values()) { - for (var summaryField : summary.getSummaryFields().values()) { - if (summaryField.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) { - summaryField.setTransform(SummaryTransform.DOCUMENT_ID); - } - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java new file mode 100644 index 00000000000..7988a0b9ceb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java @@ -0,0 +1,50 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/* + * Check that summary fields with summary transform 'tokens' have a source field with a data type that is one of + * string, array<string> or weightedset<string>. + */ +public class TokensTransformValidator extends Processor { + public TokensTransformValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (!validate || documentsOnly) { + return; + } + for (var summary : schema.getSummaries().values()) { + for (var summaryField : summary.getSummaryFields().values()) { + if (summaryField.getTransform().isTokens()) { + var source = summaryField.getSingleSource(); + if (source != null) { + var field = schema.getField(source); + if (field != null) { + var type = field.getDataType(); + var innerType = type.getPrimitiveType(); + if (innerType != DataType.STRING) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', document-summary '" + summary.getName() + + "', summary field '" + summaryField.getName() + + "', source field '" + field.getName() + + "', source field type '" + type.getName() + + "': transform '" + SummaryTransform.TOKENS.getName() + + "' is only allowed for fields of type" + + " string, array<string> or weightedset<string>"); + } + } + } + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java index dd2fd72b280..662f3fc970b 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java @@ -49,7 +49,9 @@ public class ValidateFieldTypes extends Processor { final protected void verifySummaryFields(String searchName, Map<String, DataType> seenFields) { for (DocumentSummary summary : schema.getSummaries().values()) { for (SummaryField field : summary.getSummaryFields().values()) { - checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields); + if (!field.hasUnresolvedType()) { + checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields); + } } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 2a316c8af60..1c53ee36497 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -66,6 +66,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** True if this field was defined implicitly */ private boolean implicit = false; + private boolean unresolvedType = false; /** Creates a summary field with NONE as transform */ public SummaryField(String name, DataType type) { @@ -87,10 +88,24 @@ public class SummaryField extends Field implements Cloneable, TypedKey { this.transform=transform; } + public static SummaryField createWithUnresolvedType(String name) { + /* + * Data type is not available during conversion of + * parsed schema to schema. Use a placeholder data type and tag the summary + * field as having an unresolved type. + */ + var summaryField = new SummaryField(name, DataType.NONE); + summaryField.unresolvedType = true; + return summaryField; + } + + public void setImplicit(boolean implicit) { this.implicit=implicit; } public boolean isImplicit() { return implicit; } + public boolean hasUnresolvedType() { return unresolvedType; } + public void setTransform(SummaryTransform transform) { this.transform = transform; if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { @@ -246,6 +261,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { clone.sources = new LinkedHashSet<>(this.sources); if (this.destinations != null) clone.destinations = new LinkedHashSet<>(destinations); + clone.unresolvedType = unresolvedType; return clone; } catch (CloneNotSupportedException e) { @@ -272,6 +288,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey { return true; } + public void setResolvedDataType(DataType type) { + this.dataType = type; + if (!hasForcedId()) { + this.fieldId = calculateIdV7(null); + } + unresolvedType = false; + } + public VsmCommand getVsmCommand() { return vsmCommand; } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index 575a3a748e6..58f47680f9f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -23,7 +23,8 @@ public enum SummaryTransform { MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), COPY("copy"), - DOCUMENT_ID("documentid"); + DOCUMENT_ID("documentid"), + TOKENS("tokens"); private final String name; @@ -68,6 +69,8 @@ public enum SummaryTransform { return this==DYNAMICBOLDED || this==DYNAMICTEASER; } + public boolean isTokens() { return this == TOKENS; } + /** Returns whether this transform always gets its value by accessing memory only */ public boolean isInMemory() { return switch (this) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 9fab6a2b17b..348b84367d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -48,6 +48,7 @@ import java.util.stream.Stream; import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; import static ai.vespa.metrics.set.MetricSet.empty; import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.Vespa9DefaultMetricSet.vespa9defaultMetricSet; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.addMetrics; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.generateConsumers; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.toConsumerBuilder; @@ -169,8 +170,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC public MetricsConsumer newDefaultConsumer() { if (isHostedVespa()) { - // TODO: use different metric set for hosted vespa. - return MetricsConsumer.consumer(NEW_DEFAULT_CONSUMER_ID, defaultMetricSet, systemMetricSet); + return MetricsConsumer.consumer(NEW_DEFAULT_CONSUMER_ID, vespa9defaultMetricSet, systemMetricSet); } return MetricsConsumer.consumer(NEW_DEFAULT_CONSUMER_ID, defaultMetricSet, systemMetricSet); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 510fff66c10..425a662bb2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.text.Text; import com.yahoo.vespa.model.VespaModel; import java.util.logging.Level; @@ -32,21 +33,20 @@ public class JvmHeapSizeValidator extends Validator { double gbLimit = 0.6; double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); - ds.getDeployLogger().log(Level.FINE, () -> "JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB" - .formatted(mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + ds.getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", + mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); if (mp.percentage() < percentLimit) { - throw new IllegalArgumentException( - ("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + + throw new IllegalArgumentException(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + - "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") - .formatted(clusterId, mp.percentage(), percentLimit, modelCostGb)); + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", + clusterId, mp.percentage(), percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { throw new IllegalArgumentException( - ("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + + Text.format("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + - "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") - .formatted(clusterId, availableMemoryGb, gbLimit, modelCostGb)); + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", + clusterId, availableMemoryGb, gbLimit, modelCostGb)); } } }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index 50e6cace8b8..273e5580403 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.XML; import com.yahoo.config.model.producer.AnyConfigProducer; @@ -8,6 +9,8 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; +import java.util.logging.Level; + /** * Builder for the tuning config for a search cluster. * @@ -20,7 +23,7 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui Tuning tuning = new Tuning(parent); for (Element e : XML.getChildren(spec)) { if (equals("searchnode", e)) - handleSearchNode(e, tuning); + handleSearchNode(deployState.getDeployLogger(), e, tuning); } return tuning; } @@ -45,7 +48,7 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui return Double.parseDouble(e.getFirstChild().getNodeValue()); } - private void handleSearchNode(Element spec, Tuning t) { + private void handleSearchNode(DeployLogger deployLogger, Element spec, Tuning t) { t.searchNode = new Tuning.SearchNode(); for (Element e : XML.getChildren(spec)) { if (equals("requestthreads", e)) { @@ -53,13 +56,13 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } else if (equals("flushstrategy", e)) { handleFlushStrategy(e, t.searchNode); } else if (equals("resizing", e)) { - handleResizing(e, t.searchNode); + handleResizing(deployLogger, e, t.searchNode); } else if (equals("index", e)) { - handleIndex(e, t.searchNode); + handleIndex(deployLogger, e, t.searchNode); } else if (equals("attribute", e)) { - handleAttribute(e, t.searchNode); + deployLogger.logApplicationPackage(Level.WARNING, "searchnode.attribute is deprecated and ignored."); } else if (equals("summary", e)) { - handleSummary(e, t.searchNode); + handleSummary(deployLogger, e, t.searchNode); } else if (equals("initialize", e)) { handleInitialize(e, t.searchNode); } else if (equals("feeding", e)) { @@ -161,18 +164,19 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } - private void handleResizing(Element spec, Tuning.SearchNode sn) { + private void handleResizing(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { sn.resizing = new Tuning.SearchNode.Resizing(); for (Element e : XML.getChildren(spec)) { if (equals("initialdocumentcount", e)) { + deployLogger.logApplicationPackage(Level.WARNING, "resizing.initialdocumentcount is deprecated."); sn.resizing.initialDocumentCount = asInt(e); } else if (equals("amortize-count", e)) { - sn.resizing.amortizeCount = asInt(e); + deployLogger.logApplicationPackage(Level.WARNING, "resizing.amortize-count is deprecated and ignored"); } } } - private void handleIndex(Element spec, Tuning.SearchNode sn) { + private void handleIndex(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { sn.index = new Tuning.SearchNode.Index(); for (Element e : XML.getChildren(spec)) { if (equals("io", e)) { @@ -180,9 +184,9 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui Tuning.SearchNode.Index.Io io = sn.index.io; for (Element e2 : XML.getChildren(e)) { if (equals("write", e2)) { - io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); + deployLogger.logApplicationPackage(Level.WARNING, "index.io.write is deprecated and ignored."); } else if (equals("read", e2)) { - io.read = Tuning.SearchNode.IoType.fromString(asString(e2)); + deployLogger.logApplicationPackage(Level.WARNING, "index.io.read is deprecated and ignored."); } else if (equals("search", e2)) { io.search = Tuning.SearchNode.IoType.fromString(asString(e2)); } @@ -201,28 +205,14 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } - private void handleAttribute(Element spec, Tuning.SearchNode sn) { - sn.attribute = new Tuning.SearchNode.Attribute(); - for (Element e : XML.getChildren(spec)) { - if (equals("io", e)) { - sn.attribute.io = new Tuning.SearchNode.Attribute.Io(); - for (Element e2 : XML.getChildren(e)) { - if (equals("write", e2)) { - sn.attribute.io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); - } - } - } - } - } - - private void handleSummary(Element spec, Tuning.SearchNode sn) { + private void handleSummary(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { sn.summary = new Tuning.SearchNode.Summary(); for (Element e : XML.getChildren(spec)) { if (equals("io", e)) { sn.summary.io = new Tuning.SearchNode.Summary.Io(); for (Element e2 : XML.getChildren(e)) { if (equals("write", e2)) { - sn.summary.io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); + deployLogger.logApplicationPackage(Level.WARNING, "summary.io.write is deprecated and ignored."); } else if (equals("read", e2)) { sn.summary.io.read = Tuning.SearchNode.IoType.fromString(asString(e2)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 9821f3b9568..e04711a1c56 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -49,10 +49,10 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; +import static java.util.logging.Level.FINE; /** * A container cluster that is typically set up from the user application. @@ -137,7 +137,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat ? Math.min(99, deployState.featureFlags().heapSizePercentage()) : defaultHeapSizePercentageOfAvailableMemory; onnxModelCost = deployState.onnxModelCost().newCalculator( - deployState.getApplicationPackage(), deployState.getDeployLogger()); + deployState.getApplicationPackage(), deployState.getProperties().applicationId()); logger = deployState.getDeployLogger(); } @@ -166,7 +166,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger(), deployState.featureFlags(), - userConfiguredUrls); + userConfiguredUrls, + deployState.getApplicationPackage()); for (Component<?, ?> component : getAllComponents()) { files.register(component); } @@ -217,8 +218,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat double jvmHeapDeductionGb = dynamicHeapSize ? onnxModelCost.aggregatedModelCostInBytes() / (1024D * 1024 * 1024) : 0; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); - logger.log(Level.FINE, () -> "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" - .formatted(memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); + logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" + .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); } return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index aa4aa6af32c..4e3b3d1d8cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -109,7 +109,7 @@ public abstract class Container extends AbstractService implements addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext")); appendJvmOptions(jvmOmitStackTraceInFastThrowOption(deployState.featureFlags())); - addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x200000"); + addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x800000"); } protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 0af970e016a..099255975b6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -64,8 +64,8 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { clientBindings.addAll(Arrays.asList(bindings)); } - public final Set<BindingPattern> getServerBindings() { - return Collections.unmodifiableSet(serverBindings); + public final Collection<BindingPattern> getServerBindings() { + return List.copyOf(serverBindings); } public final List<BindingPattern> getClientBindings() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index d2faff7850b..b14495756c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -9,6 +9,7 @@ import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.component.SimpleComponent; import java.util.ArrayList; @@ -24,13 +25,11 @@ import java.util.TreeSet; public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { private final ContainerCluster<?> cluster; - private volatile boolean isHostedVespa; private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); private final SortedSet<String> ignoredUserAgentsList = new TreeSet<>(); public JettyHttpServer(String componentId, ContainerCluster<?> cluster, DeployState deployState) { super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null)); - this.isHostedVespa = deployState.isHosted(); this.cluster = cluster; FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(componentId); addChild(filterBindingsProviderComponent); @@ -42,8 +41,6 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro } } - public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } - public void addConnector(ConnectorFactory connectorFactory) { connectorFactories.add(connectorFactory); addChild(connectorFactory); @@ -64,10 +61,8 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .ignoredUserAgents(ignoredUserAgentsList) .searchHandlerPaths(List.of("/search")) ); - if (isHostedVespa) { - // Enable connection log hosted Vespa + if (cluster.getAllComponents().stream().anyMatch(c -> c instanceof ConnectionLogComponent)) builder.connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)); - } configureJettyThreadpool(builder); builder.stopTimeout(300); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index c75aca7a5fa..08b0398a98f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -9,7 +9,10 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.TreeSet; /** * Component specification for {@link com.yahoo.jdisc.http.server.jetty.ConnectorFactory} with hosted specific configuration. @@ -25,6 +28,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final Duration endpointConnectionTtl; private final List<String> remoteAddressHeaders; private final List<String> remotePortHeaders; + private final Set<String> knownServerNames; public static Builder builder(String name, int listenPort) { return new Builder(name, listenPort); } @@ -37,6 +41,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { this.endpointConnectionTtl = builder.endpointConnectionTtl; this.remoteAddressHeaders = List.copyOf(builder.remoteAddressHeaders); this.remotePortHeaders = List.copyOf(builder.remotePortHeaders); + this.knownServerNames = Collections.unmodifiableSet(new TreeSet<>(builder.knownServerNames)); } private static SslProvider createSslProvider(Builder builder) { @@ -70,7 +75,8 @@ public class HostedSslConnectorFactory extends ConnectorFactory { .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0) .accessLog(new ConnectorConfig.AccessLog.Builder() .remoteAddressHeaders(remoteAddressHeaders) - .remotePortHeaders(remotePortHeaders)); + .remotePortHeaders(remotePortHeaders)) + .serverName.known(knownServerNames); } @@ -89,6 +95,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { String tlsCaCertificatesPem; String tlsCaCertificatesPath; boolean tokenEndpoint; + Set<String> knownServerNames = Set.of(); private Builder(String name, int port) { this.name = name; this.port = port; } public Builder clientAuth(SslClientAuth auth) { clientAuth = auth; return this; } @@ -101,7 +108,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public Builder tokenEndpoint(boolean enable) { this.tokenEndpoint = enable; return this; } public Builder remoteAddressHeader(String header) { this.remoteAddressHeaders.add(header); return this; } public Builder remotePortHeader(String header) { this.remotePortHeaders.add(header); return this; } - + public Builder knownServerNames(Set<String> knownServerNames) { this.knownServerNames = Set.copyOf(knownServerNames); return this; } public HostedSslConnectorFactory build() { return new HostedSslConnectorFactory(this); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7653d814d8a..119a3ad18c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -3,19 +3,14 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; -import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; -import static com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType.jsonAccessLog; - /** * Builds the config model for the standalone config server. * @@ -57,12 +52,6 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { } @Override - protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { - super.addHttp(deployState, spec, cluster, context); - cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted()); - } - - @Override protected void addModelEvaluationRuntime(ApplicationContainerCluster cluster) { // Model evaluation bundles are pre-installed in the standalone container. } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 830440aaf8e..18020f5df5d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -574,7 +574,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Reader reader = file.createReader(); String certPem = IOUtils.readAll(reader); reader.close(); - List<X509Certificate> x509Certificates = X509CertificateUtils.certificateListFromPem(certPem); + List<X509Certificate> x509Certificates; + try { + x509Certificates = X509CertificateUtils.certificateListFromPem(certPem); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("File %s contains an invalid certificate".formatted(file.getPath().getRelative()), e); + } if (x509Certificates.isEmpty()) { throw new IllegalArgumentException("File %s does not contain any certificates.".formatted(file.getPath().getRelative())); } @@ -601,6 +606,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { var endpointCert = state.endpointCertificateSecrets().orElse(null); if (endpointCert != null) { builder.endpointCertificate(endpointCert); + Set<String> mtlsEndpointNames = state.getEndpoints().stream() + .filter(endpoint -> endpoint.authMethod() == ApplicationClusterEndpoint.AuthMethod.mtls) + .flatMap(endpoint -> endpoint.names().stream()) + .collect(Collectors.toSet()); + builder.knownServerNames(mtlsEndpointNames); boolean isPublic = state.zone().system().isPublic(); List<X509Certificate> clientCertificates = getClientCertificates(cluster); if (isPublic) { @@ -654,6 +664,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .remoteAddressHeader("X-Forwarded-For") .remotePortHeader("X-Forwarded-Port") .clientAuth(SslClientAuth.NEED) + .knownServerNames(tokenEndpoints) .build(); server.addConnector(connector); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index e4eaa02acd5..9729d7d806b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -12,18 +14,17 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; - import com.yahoo.yolean.Exceptions; -import java.io.File; import java.io.Serializable; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; /** * Utility methods for registering file distribution of files/paths/urls/models defined by the user. @@ -37,14 +38,17 @@ public class UserConfiguredFiles implements Serializable { private final DeployLogger logger; private final UserConfiguredUrls userConfiguredUrls; private final String unknownConfigDefinition; + private final ApplicationPackage applicationPackage; public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, ModelContext.FeatureFlags featureFlags, - UserConfiguredUrls userConfiguredUrls) { + UserConfiguredUrls userConfiguredUrls, + ApplicationPackage applicationPackage) { this.fileRegistry = fileRegistry; this.logger = logger; this.userConfiguredUrls = userConfiguredUrls; this.unknownConfigDefinition = featureFlags.unknownConfigDefinition(); + this.applicationPackage = applicationPackage; } /** @@ -69,8 +73,7 @@ public class UserConfiguredFiles implements Serializable { if (configDefinition == null) { String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config"; switch (unknownConfigDefinition) { - case "log" -> logger.logApplicationPackage(Level.INFO, message); - case "warning" -> logger.logApplicationPackage(Level.WARNING, message); + case "warning" -> logger.logApplicationPackage(WARNING, message); case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key); } return; @@ -156,9 +159,9 @@ public class UserConfiguredFiles implements Serializable { path = Path.fromString(builder.getValue()); } - File file = path.toFile(); - if (file.isDirectory() && (file.listFiles() == null || file.listFiles().length == 0)) - throw new IllegalArgumentException("Directory '" + path.getRelative() + "' is empty"); + ApplicationFile file = applicationPackage.getFile(path); + if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty())) + logger.logApplicationPackage(INFO, "Directory '" + path.getRelative() + "' is empty"); FileReference reference = registeredFiles.get(path); if (reference == null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index 5649cd51c95..0f89a839a26 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.OnnxMemoryStats; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.tensor.TensorType; @@ -45,7 +46,7 @@ public class OnnxModelProbe { String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); var jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); outputType = outputTypeFromJson(jsonOutput, outputName); - writeMemoryStats(app, modelPath, MemoryStats.fromJson(jsonOutput)); + writeMemoryStats(app, modelPath, OnnxMemoryStats.fromJson(jsonOutput)); if ( ! outputType.equals(TensorType.empty)) { writeProbedOutputType(app, modelPath, contextKey, outputType); } @@ -56,16 +57,11 @@ public class OnnxModelProbe { return outputType; } - private static void writeMemoryStats(ApplicationPackage app, Path modelPath, MemoryStats memoryStats) throws IOException { - String path = app.getFileReference(memoryStatsPath(modelPath)).getAbsolutePath(); + private static void writeMemoryStats(ApplicationPackage app, Path modelPath, OnnxMemoryStats memoryStats) throws IOException { + String path = app.getFileReference(OnnxMemoryStats.memoryStatsFilePath(modelPath)).getAbsolutePath(); IOUtils.writeFile(path, memoryStats.toJson().toPrettyString(), false); } - private static Path memoryStatsPath(Path modelPath) { - var fileName = OnnxModelInfo.asValidIdentifier(modelPath.getRelative()) + ".memory_stats"; - return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); - } - private static String createContextKey(String onnxName, Map<String, TensorType> inputTypes) { StringBuilder key = new StringBuilder().append(onnxName).append(":"); inputTypes.entrySet().stream().sorted(Map.Entry.comparingByKey()) @@ -161,14 +157,4 @@ public class OnnxModelProbe { } return jsonParser.readTree(output.toString()); } - - public record MemoryStats(long vmSize, long vmRss) { - static MemoryStats fromJson(JsonNode json) { - return new MemoryStats(json.get("vm_size").asLong(), json.get("vm_rss").asLong()); - } - JsonNode toJson() { - return jsonParser.createObjectNode().put("vm_size", vmSize).put("vm_rss", vmRss); - } - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 003cbbe78a8..2beec421faa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -24,7 +24,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { final static long MB = 1024 * 1024; public final static long GB = MB * 1024; // This is an approximate number based on observation of a node using 33G memory with 765M docs - private final static long MEMORY_COST_PER_DOCUMENT_STORE_ONLY = 46L; + private final static long MEMORY_COST_PER_DOCUMENT_DB_ONLY = 46L; private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; @@ -58,9 +58,10 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void getConfig(ProtonConfig.Documentdb.Builder builder) { + // TODO => Move this to backend to enable ignoring this setting. ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { - long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_STORE_ONLY; + long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_DB_ONLY; builder.allocation.initialnumdocs(numDocs/redundancy.readyCopies()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index e8d42b701ef..9621ddd1374 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -120,8 +120,8 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { } public static class Resizing implements ProtonConfig.Producer { + // TODO GC as soon as resource computation is moved to backend. public Integer initialDocumentCount = null; - public Integer amortizeCount = null; @Override public void getConfig(ProtonConfig.Builder builder) { @@ -130,29 +130,16 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { db.allocation.initialnumdocs(initialDocumentCount); } } - if (amortizeCount !=null) { - for (ProtonConfig.Documentdb.Builder db : builder.documentdb) { - db.allocation.amortizecount(amortizeCount); - } - } } } public static class Index implements ProtonConfig.Producer { public static class Io implements ProtonConfig.Producer { - public IoType write = null; - public IoType read = null; public IoType search = null; @Override public void getConfig(ProtonConfig.Builder builder) { - if (write != null) { - builder.indexing.write.io(ProtonConfig.Indexing.Write.Io.Enum.valueOf(write.name)); - } - if (read != null) { - builder.indexing.read.io(ProtonConfig.Indexing.Read.Io.Enum.valueOf(read.name)); - } if (search != null) { if (search.equals(IoType.POPULATE)) { builder.search.mmap.options.add(ProtonConfig.Search.Mmap.Options.POPULATE); @@ -184,38 +171,11 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { } } - public static class Attribute implements ProtonConfig.Producer { - public static class Io implements ProtonConfig.Producer { - public IoType write = null; - - public Io() {} - - @Override - public void getConfig(ProtonConfig.Builder builder) { - if (write != null) { - builder.attribute.write.io(ProtonConfig.Attribute.Write.Io.Enum.valueOf(write.name)); - } - } - } - - public Io io; - - @Override - public void getConfig(ProtonConfig.Builder builder) { - if (io != null) io.getConfig(builder); - } - - } - public static class Summary implements ProtonConfig.Producer { public static class Io { - public IoType write = null; public IoType read = null; public void getConfig(ProtonConfig.Summary.Builder builder) { - if (write != null) { - builder.write.io(ProtonConfig.Summary.Write.Io.Enum.valueOf(write.name)); - } if (read != null) { if (read.equals(IoType.POPULATE)) { builder.read.io(ProtonConfig.Summary.Read.Io.MMAP); @@ -389,7 +349,6 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { public FlushStrategy strategy = null; public Resizing resizing = null; public Index index = null; - public Attribute attribute = null; public Summary summary = null; public Initialize initialize = null; public Feeding feeding = null; @@ -402,7 +361,6 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { if (strategy != null) strategy.getConfig(builder); if (resizing != null) resizing.getConfig(builder); if (index != null) index.getConfig(builder); - if (attribute != null) attribute.getConfig(builder); if (summary != null) summary.getConfig(builder); if (initialize != null) initialize.getConfig(builder); if (feeding != null) feeding.getConfig(builder); diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index ae4c3b365d8..aef91e34239 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -201,6 +201,7 @@ TOKEN : | < FULL: "full" > | < STATIC: "static" > | < DYNAMIC: "dynamic" > +| < TOKENS: "tokens" > | < MATCHED_ELEMENTS_ONLY: "matched-elements-only" > | < SSCONTEXTUAL: "contextual" > | < SSOVERRIDE: "override" > @@ -1089,6 +1090,9 @@ void summaryInDocument(ParsedDocumentSummary docsum) : (<TYPE> type = dataType())? lbrace() { psf = new ParsedSummaryField(name, type); + if (type != null) { + psf.setHasExplicitType(); + } } (summaryItem(psf) (<NL>)*)* <RBRACE> { @@ -1128,6 +1132,7 @@ void summaryInFieldShort(ParsedField field) : <COLON> ( <DYNAMIC> { psf.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { psf.setFull(); } + | <TOKENS> { psf.setTokens(); } ) } @@ -1138,13 +1143,17 @@ void summaryInFieldLong(ParsedField field) : { String name = field.name(); ParsedType type = field.getType(); + boolean explicitType = false; ParsedSummaryField psf; } { - ( [ name = identifier() [ <TYPE> type = dataType() ] ] + ( [ name = identifier() [ <TYPE> { type = dataType(); explicitType = true; } ] ] lbrace() { psf = field.summaryFieldFor(name, type); + if (explicitType) { + psf.setHasExplicitType(); + } } (summaryItem(psf) (<NL>)*)* <RBRACE> ) } @@ -1173,6 +1182,7 @@ void summaryTransform(ParsedSummaryField field) : { } ( <DYNAMIC> { field.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { field.setFull(); } + | <TOKENS> { field.setTokens(); } ) } @@ -2765,6 +2775,7 @@ String identifier() : { } | <TERTIARY> | <TEXT> | <TO> + | <TOKENS> | <TRUE> | <TYPE> | <UCA> diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 5382e27e0b2..520f41609b2 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -150,10 +150,12 @@ Dispatch = element dispatch { DispatchGroup* } +# TODO: Deprecated, remove in Vespa 9 DispatchGroup = element group { DispatchNode+ } +# TODO: Deprecated, remove in Vespa 9 DispatchNode = element node { attribute distribution-key { xsd:nonNegativeInteger } } @@ -293,6 +295,8 @@ Group = element group { } Tuning = element tuning { + # TODO: Deprecated, remove in Vespa 9 + # Use the one under the content tag. element dispatch { element max-hits-per-partition { xsd:nonNegativeInteger }? }? & @@ -326,11 +330,13 @@ Tuning = element tuning { }? }? & element resizing { + # resizing is deprecated and will be gone on vespa 9 element initialdocumentcount { xsd:nonNegativeInteger }? & element amortize-count { xsd:nonNegativeInteger }? }? & element index { element io { + # io.read and io.write is deprecated and will be gone on vespa 9 element write { TuningIoOptionsLight }? & element read { TuningIoOptionsLight }? & element search { TuningIoOptionsSearch }? @@ -341,12 +347,14 @@ Tuning = element tuning { }? }? & element attribute { + # attribute element is deprecated and will be gone on vespa 9 element io { element write { TuningIoOptionsLight }? } }? & element summary { element io { + # summary.io.write is deprecated and will be gone on vespa 9 element write { TuningIoOptionsLight }? & element read { TuningIoOptionsFull }? }? & @@ -354,6 +362,7 @@ Tuning = element tuning { element cache { element maxsize { xsd:nonNegativeInteger }? & element maxsize-percent { xsd:double { minInclusive = "0.0" maxInclusive = "50.0" } }? & + # initialentries is deprecated and will be gone on vespa 9 element initialentries { xsd:nonNegativeInteger }? & TuningCompression? }? & |