diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-05-09 13:32:10 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-09 13:32:10 +0200 |
commit | 8d51c1b7eb0a8a11def50b38b0cc4f41d5ce4323 (patch) | |
tree | d34024731061f0ed69b1568828d7723c03f53922 | |
parent | c72f4f8814368b461ce03759edf0b2dbe2294be0 (diff) | |
parent | 7d71cedbc90d514313e03dceffb1788ab6c033f1 (diff) |
Merge pull request #22511 from vespa-engine/bratseth/input-defaults
Bratseth/input defaults
30 files changed, 900 insertions, 382 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 9c802075462..e1fe795d2b1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -24,9 +24,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.Arguments; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import java.io.File; import java.io.IOException; import java.io.Reader; import java.io.Serializable; @@ -116,7 +116,7 @@ public class RankProfile implements Cloneable { // This cache must be invalidated every time modifications are done to 'functions'. private CachedFunctions allFunctionsCached = null; - private Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private Map<Reference, Input> inputs = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -764,33 +764,31 @@ public class RankProfile implements Cloneable { * All inputs must either be declared through this or in query profile types, * otherwise they are assumes to be scalars. */ - public void addInput(Reference reference, TensorType declaredType) { + public void addInput(Reference reference, Input input) { if (inputs.containsKey(reference)) { - TensorType hadType = inputs().get(reference); - if (! declaredType.equals(hadType)) - throw new IllegalArgumentException("Duplicate input '" + name + "' declared with both type " + - hadType + " and " + declaredType); + Input existing = inputs().get(reference); + if (! input.equals(existing)) + throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing"); } - inputs.put(reference, declaredType); + inputs.put(reference, input); } /** Returns the inputs of this, which also includes all inputs of the parents of this. */ // This is less restrictive than most other constructs in allowing inputs to be defined in all parent profiles // because inputs are tied closer to functions than the profile itself. - public Map<Reference, TensorType> inputs() { + public Map<Reference, Input> inputs() { if (inputs.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(inputs); // Combine - Map<Reference, TensorType> allInputs = new LinkedHashMap<>(); + Map<Reference, Input> allInputs = new LinkedHashMap<>(); for (var inheritedProfile : inherited()) { for (var input : inheritedProfile.inputs().entrySet()) { - TensorType existingType = allInputs.get(input.getKey()); - if (existingType != null && ! existingType.equals(input.getValue())) + Input existing = allInputs.get(input.getKey()); + if (existing != null && ! existing.equals(input.getValue())) throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + - input.getValue() + ", with type " + input.getValue() + "" + - " but this input is already defined with type " + existingType + - " in another profile this inherits"); + input.getValue() + ", but this input is already defined as " + + existing + " in another profile this inherits"); allInputs.put(input.getKey(), input.getValue()); } } @@ -1050,7 +1048,8 @@ public class RankProfile implements Cloneable { public MapEvaluationTypeContext typeContext() { return typeContext(new QueryProfileRegistry()); } private Map<Reference, TensorType> featureTypes() { - Map<Reference, TensorType> featureTypes = new HashMap<>(inputs()); + Map<Reference, TensorType> featureTypes = inputs().values().stream() + .collect(Collectors.toMap(input -> input.name(), input -> input.type())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); return featureTypes; @@ -1070,7 +1069,7 @@ public class RankProfile implements Cloneable { TensorType type = field.getType().asTensorType(); Optional<Reference> feature = Reference.simple(field.getName()); if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue; - if (featureTypes.containsKey(feature)) continue; // Explicit feature types (from inputs) overrides + if (featureTypes.containsKey(feature.get())) continue; // Explicit feature types (from inputs) overrides TensorType existingType = context.getType(feature.get()); if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) @@ -1392,6 +1391,46 @@ public class RankProfile implements Cloneable { } + public static final class Input { + + private final Reference name; + private final TensorType type; + private final Optional<Tensor> defaultValue; + + public Input(Reference name, TensorType type, Optional<Tensor> defaultValue) { + this.name = name; + this.type = type; + this.defaultValue = defaultValue; + } + + public Reference name() { return name; } + public TensorType type() { return type; } + public Optional<Tensor> defaultValue() { return defaultValue; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Input)) return false; + Input other = (Input)o; + if ( ! other.name().equals(this.name())) return false; + if ( ! other.type().equals(this.type())) return false; + if ( ! other.defaultValue().equals(this.defaultValue())) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, defaultValue); + } + + @Override + public String toString() { + return "input '" + name + "' " + type + + (defaultValue().isPresent() ? ":" + defaultValue.get().toAbbreviatedString() : ""); + } + + } + private static class CachedFunctions { private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 3c14a2b9c63..77245da5ddd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -145,7 +145,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); private final Map<String, String> attributeTypes; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; private final Set<String> filterFields = new java.util.LinkedHashSet<>(); private final String rankprofileName; @@ -426,10 +426,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } - for (Map.Entry<Reference, TensorType> input : inputs.entrySet()) { - if (FeatureNames.isQueryFeature(input.getKey())) - properties.add(new Pair<>("vespa.type.query." + input.getKey().arguments().expressions().get(0), - input.getValue().toString())); + for (var input : inputs.values()) { + if (FeatureNames.isQueryFeature(input.name())) { + properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), + input.type().toString())); + if (input.defaultValue().isPresent()) + properties.add(new Pair<>(input.name().toString(), + input.type().rank() == 0 ? + String.valueOf(input.defaultValue().get().asDouble()) : + input.defaultValue().get().toString(false, false))); + } } if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); distributeLargeExpressionsAsFiles(properties, largeRankExpressions); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java index eeb3a97eda9..0c1e5a76a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -15,6 +16,7 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; /** * Information about a schema. @@ -100,7 +102,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } schemaBuilder.rankprofile(rankProfileConfig); @@ -113,7 +115,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { this.name = profile.name(); @@ -125,7 +127,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } - public Map<Reference, TensorType> inputs() { return inputs; } + public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 118945369d3..63a120f7c7b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; +import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; import com.yahoo.searchdefinition.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; @@ -52,7 +53,7 @@ class ParsedRankProfile extends ParsedBlock { private final Map<String, String> fieldsRankType = new LinkedHashMap<>(); private final Map<String, List<String>> rankProperties = new LinkedHashMap<>(); private final Map<String, Value> constants = new LinkedHashMap<>(); - private final Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private final Map<Reference, RankProfile.Input> inputs = new LinkedHashMap<>(); ParsedRankProfile(String name) { super(name, "rank-profile"); @@ -83,7 +84,7 @@ class ParsedRankProfile extends ParsedBlock { Map<String, String> getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } Map<String, List<String>> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } Map<String, Value> getConstants() { return Collections.unmodifiableMap(constants); } - Map<Reference, TensorType> getInputs() { return Collections.unmodifiableMap(inputs); } + Map<Reference, RankProfile.Input> getInputs() { return Collections.unmodifiableMap(inputs); } Optional<String> getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } @@ -110,9 +111,9 @@ class ParsedRankProfile extends ParsedBlock { constants.put(name, value); } - void addInput(Reference name, TensorType type) { + void addInput(Reference name, RankProfile.Input input) { verifyThat(! inputs.containsKey(name), "already has input", name); - inputs.put(name, type); + inputs.put(name, input); } void addFieldRankFilter(String field, boolean filter) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 9de6a11ce44..fb7e67f2aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -97,7 +97,8 @@ public class RankProfileTypeSettingsProcessor extends Processor { private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) { for (RankProfile profile : rankProfileRegistry.all()) { if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence - profile.addInput(queryFeature, queryFeatureType); + profile.addInput(queryFeature, + new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 8cbd94a8a49..edd269559ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -215,7 +215,8 @@ public class ConvertedModel { for (ImportedMlFunction outputFunction : model.outputExpressions()) { ExpressionFunction expression = asExpressionFunction(outputFunction); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference name = Reference.fromIdentifier(input.getKey()); + profile.addInput(name, new RankProfile.Input(name, input.getValue(), Optional.empty())); } addExpression(expression, expression.getName(), constantsReplacedByFunctions, store, profile, queryProfiles, expressions); @@ -283,7 +284,8 @@ public class ConvertedModel { String name = output.getFirst(); ExpressionFunction expression = output.getSecond(); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference inputName = Reference.fromIdentifier(input.getKey()); + profile.addInput(inputName, new RankProfile.Input(inputName, input.getValue(), Optional.empty())); } TensorType type = expression.getBody().type(profile.typeContext()); if (type != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index d7ce64d1d32..1141af6d79d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -120,7 +120,7 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } docDbBuilder.rankprofile(rankProfileConfig); diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index d2d27a7a2d4..6ce261ebb8c 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1,9 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Duplicate Schema parser - work in progress -// @author arnej27959 -// NOTE: When grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf - options { UNICODE_INPUT = true; CACHE_TOKENS = false; @@ -43,7 +39,10 @@ import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorAddress; import java.util.Optional; import java.util.Map; @@ -55,6 +54,8 @@ import java.util.logging.Level; /** * The schema parser * + * NOTE: When this grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf + * * @author bratseth */ public class IntermediateParser { @@ -62,9 +63,7 @@ public class IntermediateParser { private DeployLogger deployLogger; private ModelContext.Properties properties; - /** - * Creates a parser - */ + /** Creates a parser. */ public IntermediateParser(SimpleCharStream stream, DeployLogger deployLogger, ModelContext.Properties properties) @@ -266,7 +265,7 @@ TOKEN : | < FASTRANK: "fast-rank" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > -| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < COMPRESSION: "compression" > @@ -303,6 +302,7 @@ TOKEN : | < ENABLE_BM25: "enable-bm25" > | < HNSW: "hnsw" > | < MAXLINKSPERNODE: "max-links-per-node" > +| < DOUBLE_KEYWORD: "double" > | < DISTANCEMETRIC: "distance-metric" > | < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < MULTITHREADEDINDEXING: "multi-threaded-indexing" > @@ -340,7 +340,8 @@ TOKEN : | < URI: "uri" > | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* > | < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* > -| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" > | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > | < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ > | < INTEGER: ("-")? (["0"-"9"])+ > @@ -352,8 +353,8 @@ TOKEN : | < LESSTHAN: "<" > | < GREATERTHAN: ">" > | < VARIABLE: "$" <IDENTIFIER> > -| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > -| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > } // Declare a special skip token for comments. @@ -1850,7 +1851,7 @@ void rankProfileItem(ParsedRankProfile profile) : { } } /** - * This rule consumes an inherits statement of a rank-profile. + * Consumes an inherits statement of a rank-profile. * * @param profile the profile to modify */ @@ -2067,10 +2068,35 @@ void inputs(ParsedRankProfile profile) : } { <INPUTS> <LBRACE> (<NL>)* - ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) * + ( input(profile) (<NL>)*) * <RBRACE> } +void input(ParsedRankProfile profile) : +{ + Reference reference; + TensorType type; + Tensor defaultValue = null; +} +{ + reference = queryFeature() type = inputType(reference) ( <COLON> (<NL>)* defaultValue = tensorValue(type) )? + { profile.addInput(reference, new RankProfile.Input(reference, type, Optional.ofNullable(defaultValue))); } +} + +TensorType inputType(Reference reference) : +{ + TensorType type; + +} +{ + ( + ( type = tensorType("Type of " + reference) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } +} + Reference queryFeature() : { String argument; @@ -2243,7 +2269,7 @@ void approximateThreshold(ParsedRankProfile profile) : } /** - * This rule consumes a rank-properties block of a rank profile. There + * Consumes a rank-properties block of a rank profile. There * is a little trick within this rule to allow the final rank property * to skip the terminating newline token. * @@ -2256,7 +2282,7 @@ void rankProperties(ParsedRankProfile profile) : { } } /** - * This rule consumes a single rank property pair for a rank profile. + * Consumes a single rank property pair for a rank profile. * * @param profile the rank profile to modify */ @@ -2270,25 +2296,28 @@ void rankProperty(ParsedRankProfile profile) : } /** - * This rule consumes a single rank property for a rank-properties block. + * Consumes a single rank property for a rank-properties block. * - * @return The token image of the consumed item. + * @return the token image of the consumed item */ String rankPropertyItem() : { - String image, ret = ""; + String image = null; + String ret = ""; + Token dToken = null; } { - ( ( image = identifierWithDash() { ret += image; } + ( ( image = identifierWithDash() { ret += image; } + | dToken = <DOUBLE> { ret += dToken.image; } | image = quotedString() { ret += image; } | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ ) { return ret; } } /** - * This rule consumes a field-weight statement of a rank profile. + * Consumes a field-weight statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldWeight(ParsedRankProfile profile) : { @@ -2301,9 +2330,9 @@ void fieldWeight(ParsedRankProfile profile) : } /** - * This rule consumes a rank-type statement of a rank profile. + * Consumes a rank-type statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankType(ParsedRankProfile profile) : { @@ -2316,9 +2345,9 @@ void fieldRankType(ParsedRankProfile profile) : } /** - * This rule consumes a rank filter statement of a rank profile. + * Consumes a rank filter statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankFilter(ParsedRankProfile profile) : { @@ -2330,7 +2359,7 @@ void fieldRankFilter(ParsedRankProfile profile) : } /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinSize() : { @@ -2343,7 +2372,7 @@ void rankDegradationBinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinLow() : { @@ -2354,9 +2383,8 @@ void rankDegradationBinLow() : { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); } } - /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationPosbinSize() : { @@ -2369,7 +2397,7 @@ void rankDegradationPosbinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationItem() : { } { @@ -2379,7 +2407,7 @@ void rankDegradationItem() : { } } /** - * This rule consumes a rank-degradation statement of a rank profile. + * Consumes a rank-degradation statement of a rank profile. */ void rankDegradation() : { @@ -2408,10 +2436,10 @@ void constants(ParsedRankProfile profile) : void constantValue(ParsedRankProfile profile, String name) : { - String value; + Token value; } { - <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); } } void constantTensor(ParsedRankProfile profile, String name) : @@ -2421,7 +2449,7 @@ void constantTensor(ParsedRankProfile profile, String name) : } { <LBRACE> (<NL>)* - (( tensorString = tensorValue() | + (( tensorString = tensorValuePrefixedByValue() | tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE> { if (tensorType != null) { @@ -2437,7 +2465,145 @@ String constantTensorErrorMessage(String rankProfileName, String constantTensorN { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; } } -String tensorValue() : +/** + * Parses a tensor written in a tensor literal form, + * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form + */ +Tensor tensorValue(TensorType type) : +{ + Tensor.Builder builder = Tensor.Builder.of(type); + Number doubleValue = null; +} +{ + ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = consumeNumber() ) + { + if (doubleValue != null) { + if (type.rank() > 0) + throw new IllegalArgumentException("A tensor of type " + type + " cannot be a number"); + builder.cell(doubleValue.doubleValue()); + } + return builder.build(); + } +} + +/** A mapped or mixed tensor value. */ +void mappedTensorValue(Tensor.Builder builder) : {} +{ + "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}" +} + + +void mappedTensorBlock(Tensor.Builder builder) : +{ + TensorAddress mappedAddress; +} +{ + mappedAddress = tensorAddress(builder.type().mappedSubtype()) <COLON> (<NL>)* + ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) ) +} + +void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + MixedTensor.BoundBuilder boundBuilder = (MixedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.block(sparseAddress, arrayValues); + } +} + +void indexedTensorValues(Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + IndexedTensor.BoundBuilder boundBuilder = (IndexedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.fill(arrayValues); + } +} + +/** Tensor array values. Using sub-bracketing for multiple dimensions is optional and therefore ignored here. */ +void arrayTensorValues(List<Double> values) : {} +{ + "[" ( ( indexedTensorValue(values) | arrayTensorValues(values)) )* + ( <COMMA> (<NL>)* ( indexedTensorValue(values) | arrayTensorValues(values)) )* + "]" +} + +void indexedTensorValue(List<Double> values) : +{ + Number value; +} +{ + value = consumeNumber() + { values.add(value.doubleValue()); } +} + +void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) : +{ + double value; +} +{ + value = tensorCellValue() + { builder.cell(address, value); } +} + +TensorAddress tensorAddress(TensorType type) : +{ + TensorAddress.Builder builder = new TensorAddress.Builder(type); + String label; +} +{ + ( + label = tensorAddressLabel() { builder.add(label); } + | + ( "{" ( tensorAddressElement(builder) )* ( <COMMA> tensorAddressElement(builder) )* "}" ) + ) + { return builder.build(); } +} + +void tensorAddressElement(TensorAddress.Builder builder) : +{ + String dimension; + String label; +} +{ + dimension = identifier() <COLON> (<NL>)* label = tensorAddressLabel() + { builder.add(dimension, label); } +} + +String tensorAddressLabel() : +{ + String label; +} +{ + ( label = identifier() | label = quotedString() ) + { return label; } +} + +double tensorCellValue() : +{ + Number value; +} +{ + value = consumeNumber() + { return value.doubleValue(); } +} + +/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */ +String tensorValuePrefixedByValue() : { String tensor; } @@ -2462,7 +2628,7 @@ TensorType tensorType(String errorMessage) : String tensorTypeString; } { - ( <TENSOR_TYPE> ) { tensorTypeString = token.image; } + <TENSOR_TYPE> { tensorTypeString = token.image; } { TensorType tensorType; try { @@ -2555,7 +2721,7 @@ String identifier() : { } | <DIRECT> | <DOCUMENT> | <DOCUMENTSUMMARY> - | <DOUBLE> + | <DOUBLE_KEYWORD> | <DYNAMIC> | <ENABLEBITVECTORS> | <ENABLEONLYBITVECTOR> @@ -2683,39 +2849,28 @@ String string() : { } * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can * contain anything but a double quote. * - * @return The unquoted token image. + * @return the unquoted token image */ String quotedString() : { } { - <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); } + ( <DOUBLEQUOTEDSTRING> | <SINGLEQUOTEDSTRING> ) + { return token.image.substring(1, token.image.length() - 1); } } -/** - * This rule consumes a boolean value. - * - * @return The consumed boolean value. - */ +/** A boolean value. */ Boolean bool() : { } { ( ( <ON> | <TRUE> ) { return true; } | ( <OFF> | <FALSE> ) { return false; } ) } -/** - * This rule consumes an integer token and returns its numeric value. - * - * @return The consumed integer value. - */ +/** Consumes an integer token and returns its numeric value. */ int integer() : { } { <INTEGER> { return Integer.parseInt(token.image); } } -/** - * This rule consumes a long or integer token and returns its numeric value. - * - * @return The consumed long value. - */ +/** Consumes a long or integer token and returns its numeric value. */ long consumeLong() : { } { ( <INTEGER> { return Long.parseLong(token.image); } | @@ -2723,11 +2878,7 @@ long consumeLong() : { } ) } -/** - * This rule consumes a floating-point token and returns its numeric value. - * - * @return The consumed value. - */ +/** Consumes a floating-point token and returns its numeric value. */ double consumeFloat() : { } { <DOUBLE> { return Double.valueOf(token.image); } @@ -2741,9 +2892,7 @@ Number consumeNumber() : (num = consumeFloat() | num = consumeLong()) { return num; } } -/** - * This rule consumes an opening brace with leading and trailing newline tokens. - */ +/** Consumes an opening brace with leading and trailing newline tokens. */ void lbrace() : { } { (<NL>)* <LBRACE> (<NL>)* diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 0ff9513885f..de2ce3d6938 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -202,6 +202,7 @@ TOKEN : | < INDEXING: "indexing" > | < SUMMARYTO: "summary-to" > | < DOCUMENTSUMMARY: "document-summary" > +| <DOUBLE_KEYWORD: "double" > | < RANKTYPE: "rank-type" > | < WEIGHT: "weight" > | < TYPE: "type" > @@ -296,7 +297,7 @@ TOKEN : | < PAGED: "paged" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > -| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < COMPRESSION: "compression" > @@ -2196,15 +2197,36 @@ void secondPhaseItem(RankProfile profile) : } /** Consumes an inputs block of a rank profile. */ -void inputs(RankProfile profile) : +void inputs(RankProfile profile) : {} +{ + <INPUTS> <LBRACE> (<NL>)* + ( input(profile) (<NL>)*) * + <RBRACE> +} + +void input(RankProfile profile) : { Reference reference; TensorType type; } { - <INPUTS> <LBRACE> (<NL>)* - ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) * - <RBRACE> + reference = queryFeature() type = inputType(reference) + { + profile.addInput(reference, new RankProfile.Input(reference, type, Optional.empty())); + } +} + +TensorType inputType(Reference reference) : +{ + TensorType type; +} +{ + ( + ( type = tensorType("Type of " + reference) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } } Reference queryFeature() : @@ -2408,10 +2430,13 @@ void rankProperty(RankProfile profile) : */ String rankPropertyItem() : { - String image, ret = ""; + String image; + String ret = ""; + Token dToken; } { - ( ( image = identifierWithDash() { ret += image; } + ( ( image = identifierWithDash() { ret += image; } + | dToken = <DOUBLE> { ret += dToken.image; } | image = quotedString() { ret += image; } | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ ) { return ret; } @@ -2542,10 +2567,10 @@ void constants(RankProfile profile) : void constantValue(RankProfile profile, String name) : { - String value; + Token value; } { - <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); } } void constantTensor(RankProfile profile, String name) : @@ -2596,7 +2621,7 @@ TensorType tensorType(String errorMessage) : String tensorTypeString; } { - ( <TENSOR_TYPE> ) { tensorTypeString = token.image; } + <TENSOR_TYPE> { tensorTypeString = token.image; } { TensorType tensorType; try { @@ -2693,7 +2718,7 @@ String identifier() : { } | <DIRECT> | <DOCUMENT> | <DOCUMENTSUMMARY> - | <DOUBLE> + | <DOUBLE_KEYWORD> | <DYNAMIC> | <ENABLEBITVECTORS> | <ENABLEONLYBITVECTOR> diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd index 9069f59bbe3..aed80f77e6f 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd @@ -77,7 +77,7 @@ search neuralnet { query(W_1) tensor(hidden[9],out[9]) query(b_1) tensor(out[9]) query(W_out) tensor(out[9]) - query(b_out) tensor(out[1]) + query(b_out) tensor(out[1]):[1.0] } } diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg index f5134dd15f9..1e05fe77324 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg @@ -11,6 +11,8 @@ rankprofile[].fef.property[].name "vespa.type.query.W_out" rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.b_out" rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" rankprofile[].name "unranked" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "value(0)" @@ -69,18 +71,20 @@ rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScrip rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeRank" -rankprofile[].fef.property[].name "vespa.type.query.b_out" -rankprofile[].fef.property[].value "tensor(out[1])" -rankprofile[].fef.property[].name "vespa.type.query.W_out" -rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_0" +rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" rankprofile[].fef.property[].name "vespa.type.query.b_0" rankprofile[].fef.property[].value "tensor(hidden[9])" -rankprofile[].fef.property[].name "vespa.type.query.b_1" -rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.W_1" rankprofile[].fef.property[].value "tensor(hidden[9],out[9])" -rankprofile[].fef.property[].name "vespa.type.query.W_0" -rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_1" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_out" +rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" +rankprofile[].fef.property[].name "vespa.type.query.W_out" +rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].name "neuralNetworkProfile" rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScript" rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)" @@ -172,15 +176,17 @@ rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(layer_out)" rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" rankprofile[].fef.property[].value "2000" -rankprofile[].fef.property[].name "vespa.type.query.b_out" -rankprofile[].fef.property[].value "tensor(out[1])" -rankprofile[].fef.property[].name "vespa.type.query.W_out" -rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_0" +rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" rankprofile[].fef.property[].name "vespa.type.query.b_0" rankprofile[].fef.property[].value "tensor(hidden[9])" -rankprofile[].fef.property[].name "vespa.type.query.b_1" -rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.W_1" rankprofile[].fef.property[].value "tensor(hidden[9],out[9])" -rankprofile[].fef.property[].name "vespa.type.query.W_0" -rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_1" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_out" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_out" +rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" diff --git a/config-model/src/test/derived/rankproperties/rank-profiles.cfg b/config-model/src/test/derived/rankproperties/rank-profiles.cfg index 47a0438a323..3ca44288282 100644 --- a/config-model/src/test/derived/rankproperties/rank-profiles.cfg +++ b/config-model/src/test/derived/rankproperties/rank-profiles.cfg @@ -1,6 +1,8 @@ rankprofile[].name "default" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" @@ -21,8 +23,10 @@ rankprofile[].fef.property[].value "0" rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" rankprofile[].fef.property[].value "true" rankprofile[].name "child" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" diff --git a/config-model/src/test/derived/rankproperties/rankproperties.sd b/config-model/src/test/derived/rankproperties/rankproperties.sd index 24977ae645f..db259f1daec 100644 --- a/config-model/src/test/derived/rankproperties/rankproperties.sd +++ b/config-model/src/test/derived/rankproperties/rankproperties.sd @@ -28,7 +28,8 @@ search rankproperties { expression: match } rank-properties { - $test:"foo" + $test1:"foo" + query(test2): 12.3 #$weight:1 } } diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd index e26358fef5a..339d147dfd0 100644 --- a/config-model/src/test/examples/rankpropvars.sd +++ b/config-model/src/test/examples/rankpropvars.sd @@ -55,13 +55,11 @@ document music { } field artist type string { - ## index-to: a indexing: index | summary } field year type int { indexing: attribute | summary - ## index-to: y } field url type uri {} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java index 86ccb816c10..0f29d2dda40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java @@ -10,8 +10,10 @@ import java.io.IOException; * @author bratseth */ public class RankPropertiesTestCase extends AbstractExportingTestCase { + @Test public void testRankProperties() throws IOException, ParseException { assertCorrectDeriving("rankproperties"); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index bcb197ed540..0b7dbf2eb21 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -435,7 +435,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } private VespaModel getIndexedSearchVespaModel(String xml) { - List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist"); + List<String> sds = generateSchemas("music", "album", "artist"); return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); } @@ -501,4 +501,55 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } } + public static String generateSchema(String name, String field1, String field2) { + return "schema " + name + " {" + + " document " + name + " {" + + " field " + field1 + " type string {\n" + + " indexing: index | summary\n" + + " summary: dynamic\n" + + " }\n" + + " field " + field2 + " type int {\n" + + " indexing: attribute | summary\n" + + " attribute: fast-access\n" + + " }\n" + + " field " + field2 + "_nfa type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " rank-profile staticrank inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }" + + " }" + + " rank-profile summaryfeatures inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " summary-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" + + " }" + + " rank-profile rankfeatures {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " rank-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " }" + + " }" + + "}"; + } + + public static List<String> generateSchemas(String ... sdNames) { + return generateSchemas(Arrays.asList(sdNames)); + } + + public static List<String> generateSchemas(List<String> sdNames) { + List<String> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2))); + i = i + 2; + } + return sds; + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 3cff04198bd..671a9f6660d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; @@ -16,11 +15,8 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; -import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; -import java.util.ArrayList; import java.util.List; import java.util.Arrays; import java.util.Map; @@ -35,92 +31,9 @@ public class DocumentDatabaseTestCase { private static final double SMALL = 0.00000000000001; - private static final String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" + - "<hosts> " + - " <host name='foo'>" + - " <alias>node0</alias>" + - " </host>" + - "</hosts>"; - - private String createVespaServices(List<String> sds, String mode) { - List<DocType> nameAndModes = new ArrayList<>(sds.size()); - for (String sd : sds) { - nameAndModes.add(DocType.create(sd, mode)); - } - return createVespaServicesXml(nameAndModes, ""); - } - private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) { - StringBuilder retval = new StringBuilder(); - retval.append("" + - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services version='1.0'>\n" + - "<admin version='2.0'>\n" + - " <adminserver hostalias='node0' />\n" + - "</admin>\n" + - "<container version='1.0'>\n" + - " <nodes>\n" + - " <node hostalias='node0'/>\n" + - " </nodes>\n" + - " <search/>\n" + - "</container>\n" + - "<content version='1.0' id='test'>\n" + - " <redundancy>1</redundancy>\n"); - retval.append(DocType.listToXml(nameAndModes)); - retval.append( - " <engine>\n" + - " <proton>\n" + - " <tuning>\n" + - " <searchnode>\n" + - xmlTuning + - " </searchnode>\n" + - " </tuning\n>" + - " </proton\n>" + - " </engine\n>" + - " <nodes>\n" + - " <node hostalias='node0' distribution-key='0'/>\n" + - " </nodes>\n" + - " </content>\n" + - "</services>\n"); - return retval.toString(); - } - - private ProtonConfig getProtonCfg(ContentSearchCluster cluster) { - ProtonConfig.Builder pb = new ProtonConfig.Builder(); - cluster.getConfig(pb); - return new ProtonConfig(pb); - } - - private void assertSingleSD(String mode) { - final List<String> sds = Arrays.asList("type1"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(); - IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); - ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); - String type1Id = "test/search/cluster.test/type1"; - ProtonConfig proton = getProtonCfg(contentSearchCluster); - assertEquals(1, proton.documentdb().size()); - assertEquals("type1", proton.documentdb(0).inputdoctypename()); - assertEquals(type1Id, proton.documentdb(0).configid()); - } - @Test public void requireThatWeCanHaveOneSDForIndexedMode() { - assertSingleSD("index"); - } - - private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { - return createModel(nameAndModes, xmlTuning, null); - } - - private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { - List<String> sds = new ArrayList<>(nameAndModes.size()); - for (DocType nameAndMode : nameAndModes) { - sds.add(nameAndMode.getType()); - } - var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), - ApplicationPackageUtils.generateSchemas(sds)); - return builder != null ? creator.create(builder) : creator.create(); + new SchemaTester().assertSingleSD("index"); } @Test @@ -178,9 +91,10 @@ public class DocumentDatabaseTestCase { if (featureFlagConcurrency != null) { properties.setFeedConcurrency(featureFlagConcurrency); } - VespaModel model = createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); + var tester = new SchemaTester(); + VespaModel model = tester.createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(global, proton.feeding().concurrency(), SMALL); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { @@ -190,11 +104,12 @@ public class DocumentDatabaseTestCase { @Test public void requireThatModeIsSet() { - VespaModel model = createModel(Arrays.asList(DocType.create("a", "index"), - DocType.create("b", "streaming"), - DocType.create("c", "store-only")), ""); + var tester = new SchemaTester(); + VespaModel model = tester.createModel(Arrays.asList(DocType.create("a", "index"), + DocType.create("b", "streaming"), + DocType.create("c", "store-only")), ""); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode()); assertEquals("a", proton.documentdb(0).inputdoctypename()); @@ -205,10 +120,11 @@ public class DocumentDatabaseTestCase { } private void verifyInitialDocumentCount(List<DocType> nameAndModes, String xmlTuning, List<Long> local) { + var tester = new SchemaTester(); assertEquals(nameAndModes.size(), local.size()); - VespaModel model = createModel(nameAndModes, xmlTuning); + VespaModel model = tester.createModel(nameAndModes, xmlTuning); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { assertEquals(local.get(i).longValue(), proton.documentdb(i).allocation().initialnumdocs()); @@ -240,14 +156,14 @@ public class DocumentDatabaseTestCase { assertEquals(attributeField, acfg.attribute(0).name()); assertEquals(attributeField+"_nfa", acfg.attribute(1).name()); RankProfilesConfig rcfg = model.getConfig(RankProfilesConfig.class, configId); - assertEquals(7, rcfg.rankprofile().size()); + assertEquals(6, rcfg.rankprofile().size()); } @Test public void testMultipleSchemas() { List<String> sds = List.of("type1", "type2", "type3"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"), - ApplicationPackageUtils.generateSchemas(sds)).create(); + var tester = new SchemaTester(); + var model = tester.createModel(sds); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); String type1Id = "test/search/cluster.test/type1"; @@ -255,7 +171,7 @@ public class DocumentDatabaseTestCase { String type3Id = "test/search/cluster.test/type3"; { assertEquals(3, indexedSearchCluster.getDocumentDbs().size()); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals("type1", proton.documentdb(0).inputdoctypename()); assertEquals(type1Id, proton.documentdb(0).configid()); @@ -295,9 +211,16 @@ public class DocumentDatabaseTestCase { @Test public void requireThatRelevantConfigIsAvailableForClusterSearcher() { - List<String> schemas = Arrays.asList("type1", "type2"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - ApplicationPackageUtils.generateSchemas(schemas)).create(); + String inputsProfile = + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " }" + + " }"; + List<String> schemas = List.of("type1", "type2"); + var tester = new SchemaTester(); + VespaModel model = tester.createModelWithRankProfile(inputsProfile, schemas); String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher"; { // documentdb-info config @@ -309,13 +232,13 @@ public class DocumentDatabaseTestCase { assertEquals("type1", db.name()); assertEquals(7, db.rankprofile().size()); - assertRankProfile(db, 0, "default", false, false); - assertRankProfile(db, 1, "unranked", false, false); - assertRankProfile(db, 2, "staticrank", false, false); - assertRankProfile(db, 3, "summaryfeatures", true, false); - assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); - assertRankProfile(db, 5, "rankfeatures", false, true); - var inputs = assertRankProfile(db, 6, "inputs", false, false); + tester.assertRankProfile(db, 0, "default", false, false); + tester.assertRankProfile(db, 1, "unranked", false, false); + tester.assertRankProfile(db, 2, "staticrank", false, false); + tester.assertRankProfile(db, 3, "summaryfeatures", true, false); + tester.assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); + tester.assertRankProfile(db, 5, "rankfeatures", false, true); + var inputs = tester.assertRankProfile(db, 6, "inputs", false, false); assertEquals(2, inputs.input().size()); assertEquals("query(foo)", inputs.input(0).name()); @@ -326,8 +249,8 @@ public class DocumentDatabaseTestCase { assertEquals(2, db.summaryclass().size()); assertEquals("default", db.summaryclass(0).name()); assertEquals("attributeprefetch", db.summaryclass(1).name()); - assertSummaryField(db, 0, 0, "f1", "longstring", true); - assertSummaryField(db, 0, 1, "f2", "integer", false); + tester.assertSummaryField(db, 0, 0, "f1", "longstring", true); + tester.assertSummaryField(db, 0, 1, "f2", "integer", false); } { // type2 DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(1); @@ -345,89 +268,10 @@ public class DocumentDatabaseTestCase { } } - /** Schema-info should contain all schemas, independent of clusters. */ - @Test - public void requireThatSchemaInfoIsAvailable() { - List<String> schemas = Arrays.asList("type1", "type2"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - ApplicationPackageUtils.generateSchemas(schemas)).create(); - assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model); - assertSchemaInfo("container", model); - } - - private void assertSchemaInfo(String configId, VespaModel model) { - { // schema-info config - SchemaInfoConfig dcfg = model.getConfig(SchemaInfoConfig.class, configId); - assertEquals(2, dcfg.schema().size()); - - { // type1 - SchemaInfoConfig.Schema schema = dcfg.schema(0); - assertEquals("type1", schema.name()); - - assertEquals(7, schema.rankprofile().size()); - assertRankProfile(schema, 0, "default", false, false); - assertRankProfile(schema, 1, "unranked", false, false); - assertRankProfile(schema, 2, "staticrank", false, false); - assertRankProfile(schema, 3, "summaryfeatures", true, false); - assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); - assertRankProfile(schema, 5, "rankfeatures", false, true); - var inputs = assertRankProfile(schema, 6, "inputs", false, false); - - assertEquals(2, inputs.input().size()); - assertEquals("query(foo)", inputs.input(0).name()); - assertEquals("tensor<float>(x[10])", inputs.input(0).type()); - assertEquals("query(bar)", inputs.input(1).name()); - assertEquals("tensor(key{},x[1000])", inputs.input(1).type()); - - // assertEquals(2, schema.summaryclass().size()); - // assertEquals("default", schema.summaryclass(0).name()); - // assertEquals("attributeprefetch", schema.summaryclass(1).name()); - // assertSummaryField(schema, 0, 0, "f1", "longstring", true); - // assertSummaryField(schema, 0, 1, "f2", "integer", false); - } - { // type2 - SchemaInfoConfig.Schema schema = dcfg.schema(1); - assertEquals("type2", schema.name()); - } - } - } - - private DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, - int index, - String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures) { - DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index); - assertEquals(name, rankProfile.name()); - assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); - assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); - return rankProfile; - } - - private SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, - int index, - String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures) { - SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index); - assertEquals(name, rankProfile.name()); - assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); - assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); - return rankProfile; - } - - private void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, - String name, String type, boolean dynamic) { - DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); - assertEquals(name, field.name()); - assertEquals(type, field.type()); - assertEquals(dynamic, field.dynamic()); - } - private void assertDocumentDBConfigAvailableForStreaming(String mode) { - final List<String> sds = Arrays.asList("type"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(); + List<String> sds = List.of("type"); + var tester = new SchemaTester(); + var model = tester.createModelWithMode(mode, sds); DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); assertEquals(1, dcfg.documentdb().size()); @@ -452,11 +296,11 @@ public class DocumentDatabaseTestCase { Map<String, List<String>> expectedAttributesMap, DeployState.Builder builder, long expectedMaxUnCommittedMemory) { - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(builder); + var tester = new SchemaTester(); + var model = tester.createModelWithMode(mode, sds, builder); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(sds.size(), proton.documentdb().size()); for (int i = 0; i < sds.size(); i++) { assertEquals(sds.get(i), proton.documentdb(i).inputdoctypename()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java new file mode 100644 index 00000000000..cf2135aea5a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class SchemaInfoTestCase { + + /** Schema-info should contain all schemas, independent of clusters. */ + @Test + public void requireThatSchemaInfoIsAvailable() { + List.of(1.0,2.0,3.0).toArray(new Double[3]); + String inputs = + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " query(myDouble1) double: 0.5" + + " query(myDouble2) tensor()" + + " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" + + " query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" + + " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" + + " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" + + " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" + + " }" + + " }"; + List<String> schemas = List.of("type1", "type2"); + var tester = new SchemaTester(); + var model = tester.createModelWithRankProfile(inputs, schemas); + assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model, tester); + assertSchemaInfo("container", model, tester); + } + + private void assertSchemaInfo(String configId, VespaModel model, SchemaTester tester) { + { + SchemaInfoConfig schemaInfoConfig = model.getConfig(SchemaInfoConfig.class, configId); + RankProfilesConfig rankProfilesConfig = model.getConfig(RankProfilesConfig.class, "test/search/cluster.test/type1"); + + assertEquals(2, schemaInfoConfig.schema().size()); + + { // type1 + SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(0); + assertEquals("type1", schema.name()); + + assertEquals(7, schema.rankprofile().size()); + tester.assertRankProfile(schema, 0, "default", false, false); + tester.assertRankProfile(schema, 1, "unranked", false, false); + tester.assertRankProfile(schema, 2, "staticrank", false, false); + tester.assertRankProfile(schema, 3, "summaryfeatures", true, false); + tester.assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); + tester.assertRankProfile(schema, 5, "rankfeatures", false, true); + + var schemaInfoProfile = tester.assertRankProfile(schema, 6, "inputs", false, false); + assertEquals(9, schemaInfoProfile.input().size()); + var rankProfilesProfile = rankProfilesConfig.rankprofile().get(6); + assertEquals("inputs", rankProfilesProfile.name()); + assertInput("query(foo)", "tensor<float>(x[10])", null, 0, schemaInfoProfile, rankProfilesProfile); + assertInput("query(bar)", "tensor(key{},x[1000])", null, 1, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble1)", "tensor()", "0.5", 2, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble2)", "tensor()", null, 3, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMap)", "tensor(key{})", "{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myVector)", "tensor(x[3])", "{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMatrix)", "tensor(x[2],y[3])", "{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 6, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed1)", "tensor(key{},x[2])", "{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 7, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile); + + assertEquals(2, schema.summaryclass().size()); + assertEquals("default", schema.summaryclass(0).name()); + assertEquals("attributeprefetch", schema.summaryclass(1).name()); + tester.assertSummaryField(schema, 0, 0, "f1", "longstring", true); + tester.assertSummaryField(schema, 0, 1, "f2", "integer", false); + } + { // type2 + SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(1); + assertEquals("type2", schema.name()); + } + } + } + + private void assertInput(String name, String type, String defaultValue, + int index, + SchemaInfoConfig.Schema.Rankprofile schemaInfoProfile, + RankProfilesConfig.Rankprofile rankProfilesProfile) { + assertEquals(name, schemaInfoProfile.input(index).name()); + assertEquals(type, schemaInfoProfile.input(index).type()); + if (defaultValue != null) { + boolean found = false; + for (var property : rankProfilesProfile.fef().property()) { + if (property.name().equals(name)) { + assertEquals(defaultValue, property.value()); + found = true; + } + } + if ( ! found) + fail("Missing property " + name); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java new file mode 100644 index 00000000000..7a077c1a0aa --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java @@ -0,0 +1,220 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.content.utils.DocType; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SchemaTester { + + private static final String vespaHosts = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<hosts> " + + " <host name='foo'>" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + private String createVespaServices(List<String> sds, String mode) { + List<DocType> nameAndModes = new ArrayList<>(sds.size()); + for (String sd : sds) { + nameAndModes.add(DocType.create(sd, mode)); + } + return createVespaServicesXml(nameAndModes, ""); + } + private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) { + StringBuilder retval = new StringBuilder(); + retval.append("" + + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + "<admin version='2.0'>\n" + + " <adminserver hostalias='node0' />\n" + + "</admin>\n" + + "<container version='1.0'>\n" + + " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " <search/>\n" + + "</container>\n" + + "<content version='1.0' id='test'>\n" + + " <redundancy>1</redundancy>\n"); + retval.append(DocType.listToXml(nameAndModes)); + retval.append( + " <engine>\n" + + " <proton>\n" + + " <tuning>\n" + + " <searchnode>\n" + + xmlTuning + + " </searchnode>\n" + + " </tuning\n>" + + " </proton\n>" + + " </engine\n>" + + " <nodes>\n" + + " <node hostalias='node0' distribution-key='0'/>\n" + + " </nodes>\n" + + " </content>\n" + + "</services>\n"); + return retval.toString(); + } + + ProtonConfig getProtonCfg(ContentSearchCluster cluster) { + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + cluster.getConfig(pb); + return new ProtonConfig(pb); + } + + void assertSingleSD(String mode) { + List<String> sds = List.of("type1"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), + generateSchemas("", sds)).create(); + IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); + String type1Id = "test/search/cluster.test/type1"; + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(1, proton.documentdb().size()); + assertEquals("type1", proton.documentdb(0).inputdoctypename()); + assertEquals(type1Id, proton.documentdb(0).configid()); + } + + VespaModel createModel(List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas("", schemas)).create(); + } + + VespaModel createModelWithRankProfile(String rankProfile, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas(rankProfile, schemas)).create(); + } + + VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { + return createModel(nameAndModes, xmlTuning, null); + } + + VespaModel createModelWithMode(String mode, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, mode), + generateSchemas("", schemas)).create(); + + } + + VespaModel createModelWithMode(String mode, List<String> schemas, DeployState.Builder builder) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, mode), + generateSchemas("", schemas)).create(builder); + } + + VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { + List<String> sds = new ArrayList<>(nameAndModes.size()); + for (DocType nameAndMode : nameAndModes) { + sds.add(nameAndMode.getType()); + } + var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), + generateSchemas("", sds)); + return builder != null ? creator.create(builder) : creator.create(); + } + + public static String generateSchema(String name, String field1, String field2, String rankProfile) { + return "schema " + name + " {" + + " document " + name + " {" + + " field " + field1 + " type string {\n" + + " indexing: index | summary\n" + + " summary: dynamic\n" + + " }\n" + + " field " + field2 + " type int {\n" + + " indexing: attribute | summary\n" + + " attribute: fast-access\n" + + " }\n" + + " field " + field2 + "_nfa type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " rank-profile staticrank inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }" + + " }" + + " rank-profile summaryfeatures inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " summary-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" + + " }" + + " rank-profile rankfeatures {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " rank-features: attribute(" + field2 + ")" + + " }" + + rankProfile + + "}"; + } + + public static List<String> generateSchemas(String rankProfile, String ... sdNames) { + return generateSchemas(rankProfile, Arrays.asList(sdNames)); + } + + public static List<String> generateSchemas(String rankProfile, List<String> sdNames) { + List<String> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), rankProfile)); + i = i + 2; + } + return sds; + } + + DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, + int index, + String name, + boolean hasSummaryFeatures, + boolean hasRankFeatures) { + DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index); + assertEquals(name, rankProfile.name()); + assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); + return rankProfile; + } + + SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, + int index, + String name, + boolean hasSummaryFeatures, + boolean hasRankFeatures) { + SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index); + assertEquals(name, rankProfile.name()); + assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); + return rankProfile; + } + + void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, + String name, String type, boolean dynamic) { + DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); + assertEquals(name, field.name()); + assertEquals(type, field.type()); + assertEquals(dynamic, field.dynamic()); + } + + void assertSummaryField(SchemaInfoConfig.Schema schema, int summaryClassIndex, int fieldIndex, + String name, String type, boolean dynamic) { + SchemaInfoConfig.Schema.Summaryclass.Fields field = schema.summaryclass(summaryClassIndex).fields(fieldIndex); + assertEquals(name, field.name()); + assertEquals(type, field.type()); + assertEquals(dynamic, field.dynamic()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index 952de2a6f40..df611deb72a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -7,6 +7,7 @@ import java.util.List; /** * For testing purposes only. + * * @author geirst */ public class ApplicationPackageUtils { diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 00c3a1d1aae..95494190734 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -47,7 +47,7 @@ public class Coverage { protected Coverage(long docs, int nodes, boolean full, int resultSets) { this(docs, docs, nodes, resultSets, full ? FullCoverageDefinition.EXPLICITLY_FULL - : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); + : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); } private Coverage(long docs, long active, int nodes, int resultSets, FullCoverageDefinition fullReason) { @@ -168,9 +168,8 @@ public class Coverage { } /** - * An int between 0 (inclusive) and 100 (inclusive) representing how many - * percent coverage the result sets this Coverage instance contains information - * about had. + * An int between 0 (inclusive) and 100 (inclusive) representing the + * percent coverage of the result sets this instance contains information about. */ public int getResultPercentage() { if (getResultSets() == 0) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 19e0e441359..abd23c1822d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -147,7 +147,7 @@ public class QueryProfileProperties extends Properties { private String toShortString(Object value) { if ( ! (value instanceof Tensor)) return value.toString(); - return ((Tensor)value).toShortString(); + return ((Tensor)value).toAbbreviatedString(); } private Object convertByType(CompoundName name, Object value, Map<String, String> context) { diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 5074a520a4e..aa561f58d7a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -26,7 +26,7 @@ public class Coverage extends com.yahoo.container.handler.Coverage { /** * Will set number of documents present in ideal state * - * @param soonActive Number of documents active in ideal state + * @param soonActive number of documents active in ideal state * @return self for chaining */ @Beta @@ -35,10 +35,11 @@ public class Coverage extends com.yahoo.container.handler.Coverage { /** * Will set the reasons for degraded coverage as reported by vespa backend. * - * @param degradedReason Reason for degradation + * @param degradedReason reason for degradation * @return self for chaining */ public Coverage setDegradedReason(int degradedReason) { this.degradedReason = degradedReason; return this; } public Coverage setNodesTried(int nodesTried) { super.setNodesTried(nodesTried); return this; } + } diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 6044666ebf8..f7be61946ba 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -787,7 +787,10 @@ "public", "abstract" ], - "methods": [], + "methods": [ + "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(float[])", + "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(double[])" + ], "fields": [] }, "com.yahoo.tensor.IndexedTensor$Builder": { @@ -903,7 +906,8 @@ "public java.util.Map cells()", "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public bridge synthetic com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)" ], @@ -954,7 +958,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)" ], "fields": [] @@ -1046,7 +1051,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public long denseSubspaceSize()", "public static com.yahoo.tensor.TensorType createPartialType(com.yahoo.tensor.TensorType$Value, java.util.List)" @@ -1092,6 +1098,7 @@ ], "methods": [ "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, java.lang.String)", + "public com.yahoo.tensor.TensorType type()", "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, long)", "public com.yahoo.tensor.Tensor$Builder value(double)", "public com.yahoo.tensor.Tensor$Builder value(float)" @@ -1233,9 +1240,11 @@ "public java.util.List largest()", "public java.util.List smallest()", "public abstract java.lang.String toString()", - "public abstract java.lang.String toShortString()", - "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, long)", - "public static java.lang.String contentToString(com.yahoo.tensor.Tensor, long)", + "public abstract java.lang.String toString(boolean, boolean)", + "public abstract java.lang.String toAbbreviatedString()", + "public java.lang.String toShortString()", + "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, boolean, boolean, long)", + "public static java.lang.String valueToString(com.yahoo.tensor.Tensor, boolean, long)", "public abstract boolean equals(java.lang.Object)", "public abstract int hashCode()", "public static boolean equals(com.yahoo.tensor.Tensor, com.yahoo.tensor.Tensor)", @@ -1256,8 +1265,10 @@ ], "methods": [ "public void <init>(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String)", "public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String, java.lang.String)", "public com.yahoo.tensor.TensorAddress$Builder copy()", + "public com.yahoo.tensor.TensorType type()", "public com.yahoo.tensor.TensorAddress build()" ], "fields": [] diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 89eefeced56..c4316eb334a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -219,21 +219,26 @@ public abstract class IndexedTensor implements Tensor { } @Override - public String toString() { return toString(Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } - Indexes indexes = Indexes.of(dimensionSizes); + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms || type.rank() == 0 || type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - StringBuilder b = new StringBuilder(type.toString()).append(":"); + Indexes indexes = Indexes.of(dimensionSizes); + StringBuilder b = new StringBuilder(); + if (withType) + b.append(type).append(":"); indexedBlockToString(this, indexes, maxCells, b); return b.toString(); } @@ -438,7 +443,7 @@ public abstract class IndexedTensor implements Tensor { this.sizes = sizes; } - BoundBuilder fill(float[] values) { + public BoundBuilder fill(float[] values) { long index = 0; for (float value : values) { cellByDirectIndex(index++, value); @@ -446,7 +451,7 @@ public abstract class IndexedTensor implements Tensor { return this; } - BoundBuilder fill(double[] values) { + public BoundBuilder fill(double[] values) { long index = 0; for (double value : values) { cellByDirectIndex(index++, value); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index ad945ed18bf..946d8fe0f4a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -72,11 +72,18 @@ public class MappedTensor implements Tensor { public int hashCode() { return cells.hashCode(); } @Override - public String toString() { return Tensor.toStandardString(this, Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return Tensor.toStandardString(this, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { return toString(withType, shortForms, Long.MAX_VALUE); } + + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + return Tensor.toStandardString(this, withType, shortForms, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 56bd94a86e9..d2fed9b96f9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -145,23 +145,27 @@ public class MixedTensor implements Tensor { @Override public String toString() { - return toString(Long.MAX_VALUE); + return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) - return Tensor.toStandardString(this, maxCells); - if (type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms + || type.rank() == 0 + || type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty()) + || type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - return type + ":" + index.contentToString(this, maxCells); + return (withType ? type + ":" : "") + index.contentToString(this, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 06e7b010a7a..8a84e97fe05 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -32,7 +32,6 @@ import java.util.Set; import java.util.function.DoubleBinaryOperator; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; -import java.util.stream.Collectors; import static com.yahoo.tensor.functions.ScalarFunctions.Hamming; @@ -316,8 +315,22 @@ public interface Tensor { @Override String toString(); + /** + * Returns this tensor on the + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * + * @param withType whether to prefix the value by the type of this + * @param shortForms whether to use short forms where applicable, or always using the verbose form + */ + String toString(boolean withType, boolean shortForms); + /** Returns an abbreviated string representation of this tensor suitable for human-readable messages */ - String toShortString(); + String toAbbreviatedString(); + + // TODO: Remove on Vespa 8 + /** @deprecated use toAbbreviatedString */ + @Deprecated + default String toShortString() { return toAbbreviatedString(); } /** * Call this from toString in implementations to return this tensor on the @@ -325,15 +338,16 @@ public interface Tensor { * (toString cannot be a default method because default methods cannot override super methods). * * @param tensor the tensor to return the standard string format of + * @param withType whether the type should be prepended to the content * @param maxCells the max number of cells to output, after which just , "..." is output to represent the rest * of the cells * @return the tensor on the standard string format */ - static String toStandardString(Tensor tensor, long maxCells) { - return tensor.type() + ":" + contentToString(tensor, maxCells); + static String toStandardString(Tensor tensor, boolean withType, boolean shortForms, long maxCells) { + return (withType ? tensor.type() + ":" : "") + valueToString(tensor, shortForms, maxCells); } - static String contentToString(Tensor tensor, long maxCells) { + static String valueToString(Tensor tensor, boolean shortForms, long maxCells) { var cellEntries = new ArrayList<>(tensor.cells().entrySet()); cellEntries.sort(Map.Entry.comparingByKey()); if (tensor.type().dimensions().isEmpty()) { @@ -345,7 +359,7 @@ public interface Tensor { for (; i < cellEntries.size() && i < maxCells; i++) { if (i > 0) b.append(", "); - b.append(cellToString(cellEntries.get(i), tensor.type())); + b.append(cellToString(cellEntries.get(i), tensor.type(), shortForms)); } if (i == maxCells && i < tensor.size()) b.append(", ..."); @@ -353,8 +367,9 @@ public interface Tensor { return b.toString(); } - private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type) { - return (type.rank() > 1 ? cell.getKey().toString(type) : TensorAddress.labelToString(cell.getKey().label(0))) + + private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type, boolean shortForms) { + return (shortForms && type.rank() == 1 ? TensorAddress.labelToString(cell.getKey().label(0)) + : cell.getKey().toString(type) ) + ":" + cell.getValue(); } @@ -533,7 +548,7 @@ public interface Tensor { return IndexedTensor.Builder.of(type, dimensionSizes); } - /** Returns the type this is building */ + /** Returns the type of the tensor this is building */ TensorType type(); /** Return a cell builder */ @@ -578,6 +593,9 @@ public interface Tensor { return this; } + /** Returns the type of the tensor this cell is build for. */ + public TensorType type() { return tensorBuilder.type(); } + public CellBuilder label(String dimension, long label) { return label(dimension, String.valueOf(label)); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 27e752f1180..d9ab67d6c5f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -168,7 +168,7 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } - /** Supports building of a tensor address */ + /** Builder of a tensor address */ public static class Builder { private final TensorType type; @@ -184,6 +184,18 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } /** + * Adds the label to the only dimension of this. + * + * @throws IllegalArgumentException if this does not have exactly one dimension + */ + public Builder add(String label) { + if (type.rank() != 1) + throw new IllegalArgumentException("Cannot add a label without explicit dimension to a tensor of type " + type); + add(type.dimensions().get(0).name(), label); + return this; + } + + /** * Adds a label in a dimension to this. * * @return this for convenience @@ -203,6 +215,9 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { return new Builder(type, Arrays.copyOf(labels, labels.length)); } + /** Returns the type of the tensor this address is being built for. */ + public TensorType type() { return type; } + public TensorAddress build() { for (int i = 0; i < labels.length; i++) if (labels[i] == null) diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 2067d7a8492..920f8512c53 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -65,24 +65,24 @@ public class TensorTestCase { @Test public void testToShortString() { assertEquals("tensor(x[10]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", - Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toShortString()); + Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toAbbreviatedString()); assertEquals("tensor(x[14]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, ...]", - Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toShortString()); + Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toAbbreviatedString()); assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0, {d1:l1,d2:l3}:6.0, ...}", Tensor.from("{{d1:l1,d2:l1}:6, {d2:l2,d1:l1}:6, {d2:l3,d1:l1}:6, {d2:l4,d1:l1}:6, {d2:l5,d1:l1}:6," + " {d2:l6,d1:l1}:6, {d2:l7,d1:l1}:6, {d2:l8,d1:l1}:6, {d2:l9,d1:l1}:6, {d2:l2,d1:l2}:6," + - " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toShortString()); + " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},n{},x[3]):{{m:k1,n:k1,x:0}:0.0, {m:k1,n:k1,x:1}:1.0, {m:k1,n:k1,x:2}:2.0, ...}", Tensor.from("tensor(m{},n{},x[3]):" + "{{m:k1,n:k1,x:0}:0, {m:k1,n:k1,x:1}:1, {m:k1,n:k1,x:2}:2, " + " {m:k2,n:k1,x:0}:0, {m:k2,n:k1,x:1}:1, {m:k2,n:k1,x:2}:2, " + - " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toShortString()); + " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toAbbreviatedString()); assertEquals("tensor(m{},x[2],y[2]):{k1:[[0.0, 1.0], [2.0, 3.0]], k2:[[0.0, ...}", - Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toShortString()); + Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toAbbreviatedString()); } @Test |