summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2022-05-09 13:32:10 +0200
committerGitHub <noreply@github.com>2022-05-09 13:32:10 +0200
commit8d51c1b7eb0a8a11def50b38b0cc4f41d5ce4323 (patch)
treed34024731061f0ed69b1568828d7723c03f53922
parentc72f4f8814368b461ce03759edf0b2dbe2294be0 (diff)
parent7d71cedbc90d514313e03dceffb1788ab6c033f1 (diff)
Merge pull request #22511 from vespa-engine/bratseth/input-defaults
Bratseth/input defaults
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java73
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java16
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java8
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java9
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java2
-rw-r--r--config-model/src/main/javacc/IntermediateParser.jj273
-rw-r--r--config-model/src/main/javacc/SDParser.jj47
-rw-r--r--config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd2
-rw-r--r--config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg38
-rw-r--r--config-model/src/test/derived/rankproperties/rank-profiles.cfg8
-rw-r--r--config-model/src/test/derived/rankproperties/rankproperties.sd3
-rw-r--r--config-model/src/test/examples/rankpropvars.sd2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java53
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java238
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java106
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java220
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java1
-rw-r--r--container-core/src/main/java/com/yahoo/container/handler/Coverage.java7
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/result/Coverage.java5
-rw-r--r--vespajlib/abi-spec.json25
-rw-r--r--vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java27
-rw-r--r--vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java13
-rw-r--r--vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java26
-rw-r--r--vespajlib/src/main/java/com/yahoo/tensor/Tensor.java36
-rw-r--r--vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java17
-rw-r--r--vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java14
30 files changed, 900 insertions, 382 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index 9c802075462..e1fe795d2b1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -24,9 +24,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
import com.yahoo.searchlib.rankingexpression.evaluation.Value;
import com.yahoo.searchlib.rankingexpression.rule.Arguments;
import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
-import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.Serializable;
@@ -116,7 +116,7 @@ public class RankProfile implements Cloneable {
// This cache must be invalidated every time modifications are done to 'functions'.
private CachedFunctions allFunctionsCached = null;
- private Map<Reference, TensorType> inputs = new LinkedHashMap<>();
+ private Map<Reference, Input> inputs = new LinkedHashMap<>();
private Set<String> filterFields = new HashSet<>();
@@ -764,33 +764,31 @@ public class RankProfile implements Cloneable {
* All inputs must either be declared through this or in query profile types,
* otherwise they are assumes to be scalars.
*/
- public void addInput(Reference reference, TensorType declaredType) {
+ public void addInput(Reference reference, Input input) {
if (inputs.containsKey(reference)) {
- TensorType hadType = inputs().get(reference);
- if (! declaredType.equals(hadType))
- throw new IllegalArgumentException("Duplicate input '" + name + "' declared with both type " +
- hadType + " and " + declaredType);
+ Input existing = inputs().get(reference);
+ if (! input.equals(existing))
+ throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing");
}
- inputs.put(reference, declaredType);
+ inputs.put(reference, input);
}
/** Returns the inputs of this, which also includes all inputs of the parents of this. */
// This is less restrictive than most other constructs in allowing inputs to be defined in all parent profiles
// because inputs are tied closer to functions than the profile itself.
- public Map<Reference, TensorType> inputs() {
+ public Map<Reference, Input> inputs() {
if (inputs.isEmpty() && inherited().isEmpty()) return Map.of();
if (inherited().isEmpty()) return Collections.unmodifiableMap(inputs);
// Combine
- Map<Reference, TensorType> allInputs = new LinkedHashMap<>();
+ Map<Reference, Input> allInputs = new LinkedHashMap<>();
for (var inheritedProfile : inherited()) {
for (var input : inheritedProfile.inputs().entrySet()) {
- TensorType existingType = allInputs.get(input.getKey());
- if (existingType != null && ! existingType.equals(input.getValue()))
+ Input existing = allInputs.get(input.getKey());
+ if (existing != null && ! existing.equals(input.getValue()))
throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " +
- input.getValue() + ", with type " + input.getValue() + "" +
- " but this input is already defined with type " + existingType +
- " in another profile this inherits");
+ input.getValue() + ", but this input is already defined as " +
+ existing + " in another profile this inherits");
allInputs.put(input.getKey(), input.getValue());
}
}
@@ -1050,7 +1048,8 @@ public class RankProfile implements Cloneable {
public MapEvaluationTypeContext typeContext() { return typeContext(new QueryProfileRegistry()); }
private Map<Reference, TensorType> featureTypes() {
- Map<Reference, TensorType> featureTypes = new HashMap<>(inputs());
+ Map<Reference, TensorType> featureTypes = inputs().values().stream()
+ .collect(Collectors.toMap(input -> input.name(), input -> input.type()));
allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes));
allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes));
return featureTypes;
@@ -1070,7 +1069,7 @@ public class RankProfile implements Cloneable {
TensorType type = field.getType().asTensorType();
Optional<Reference> feature = Reference.simple(field.getName());
if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue;
- if (featureTypes.containsKey(feature)) continue; // Explicit feature types (from inputs) overrides
+ if (featureTypes.containsKey(feature.get())) continue; // Explicit feature types (from inputs) overrides
TensorType existingType = context.getType(feature.get());
if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get())))
@@ -1392,6 +1391,46 @@ public class RankProfile implements Cloneable {
}
+ public static final class Input {
+
+ private final Reference name;
+ private final TensorType type;
+ private final Optional<Tensor> defaultValue;
+
+ public Input(Reference name, TensorType type, Optional<Tensor> defaultValue) {
+ this.name = name;
+ this.type = type;
+ this.defaultValue = defaultValue;
+ }
+
+ public Reference name() { return name; }
+ public TensorType type() { return type; }
+ public Optional<Tensor> defaultValue() { return defaultValue; }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) return true;
+ if ( ! (o instanceof Input)) return false;
+ Input other = (Input)o;
+ if ( ! other.name().equals(this.name())) return false;
+ if ( ! other.type().equals(this.type())) return false;
+ if ( ! other.defaultValue().equals(this.defaultValue())) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, type, defaultValue);
+ }
+
+ @Override
+ public String toString() {
+ return "input '" + name + "' " + type +
+ (defaultValue().isPresent() ? ":" + defaultValue.get().toAbbreviatedString() : "");
+ }
+
+ }
+
private static class CachedFunctions {
private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index 3c14a2b9c63..77245da5ddd 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -145,7 +145,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
*/
private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default");
private final Map<String, String> attributeTypes;
- private final Map<Reference, TensorType> inputs;
+ private final Map<Reference, RankProfile.Input> inputs;
private final Set<String> filterFields = new java.util.LinkedHashSet<>();
private final String rankprofileName;
@@ -426,10 +426,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) {
properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue()));
}
- for (Map.Entry<Reference, TensorType> input : inputs.entrySet()) {
- if (FeatureNames.isQueryFeature(input.getKey()))
- properties.add(new Pair<>("vespa.type.query." + input.getKey().arguments().expressions().get(0),
- input.getValue().toString()));
+ for (var input : inputs.values()) {
+ if (FeatureNames.isQueryFeature(input.name())) {
+ properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0),
+ input.type().toString()));
+ if (input.defaultValue().isPresent())
+ properties.add(new Pair<>(input.name().toString(),
+ input.type().rank() == 0 ?
+ String.valueOf(input.defaultValue().get().asDouble()) :
+ input.defaultValue().get().toString(false, false)));
+ }
}
if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties");
distributeLargeExpressionsAsFiles(properties, largeRankExpressions);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java
index eeb3a97eda9..0c1e5a76a89 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java
@@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Schema;
import com.yahoo.searchlib.rankingexpression.Reference;
+import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
import com.yahoo.vespa.config.search.SummarymapConfig;
import com.yahoo.vespa.documentmodel.SummaryTransform;
@@ -15,6 +16,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Optional;
/**
* Information about a schema.
@@ -100,7 +102,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc
for (var input : rankProfile.inputs().entrySet()) {
var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder();
inputConfig.name(input.getKey().toString());
- inputConfig.type(input.getValue().toString());
+ inputConfig.type(input.getValue().type().toString());
rankProfileConfig.input(inputConfig);
}
schemaBuilder.rankprofile(rankProfileConfig);
@@ -113,7 +115,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc
private final String name;
private final boolean hasSummaryFeatures;
private final boolean hasRankFeatures;
- private final Map<Reference, TensorType> inputs;
+ private final Map<Reference, RankProfile.Input> inputs;
public RankProfileInfo(RankProfile profile) {
this.name = profile.name();
@@ -125,7 +127,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc
public String name() { return name; }
public boolean hasSummaryFeatures() { return hasSummaryFeatures; }
public boolean hasRankFeatures() { return hasRankFeatures; }
- public Map<Reference, TensorType> inputs() { return inputs; }
+ public Map<Reference, RankProfile.Input> inputs() { return inputs; }
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java
index 118945369d3..63a120f7c7b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.parser;
+import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings;
import com.yahoo.searchdefinition.RankProfile.MutateOperation;
import com.yahoo.searchlib.rankingexpression.FeatureList;
@@ -52,7 +53,7 @@ class ParsedRankProfile extends ParsedBlock {
private final Map<String, String> fieldsRankType = new LinkedHashMap<>();
private final Map<String, List<String>> rankProperties = new LinkedHashMap<>();
private final Map<String, Value> constants = new LinkedHashMap<>();
- private final Map<Reference, TensorType> inputs = new LinkedHashMap<>();
+ private final Map<Reference, RankProfile.Input> inputs = new LinkedHashMap<>();
ParsedRankProfile(String name) {
super(name, "rank-profile");
@@ -83,7 +84,7 @@ class ParsedRankProfile extends ParsedBlock {
Map<String, String> getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); }
Map<String, List<String>> getRankProperties() { return Collections.unmodifiableMap(rankProperties); }
Map<String, Value> getConstants() { return Collections.unmodifiableMap(constants); }
- Map<Reference, TensorType> getInputs() { return Collections.unmodifiableMap(inputs); }
+ Map<Reference, RankProfile.Input> getInputs() { return Collections.unmodifiableMap(inputs); }
Optional<String> getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); }
Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); }
@@ -110,9 +111,9 @@ class ParsedRankProfile extends ParsedBlock {
constants.put(name, value);
}
- void addInput(Reference name, TensorType type) {
+ void addInput(Reference name, RankProfile.Input input) {
verifyThat(! inputs.containsKey(name), "already has input", name);
- inputs.put(name, type);
+ inputs.put(name, input);
}
void addFieldRankFilter(String field, boolean filter) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
index 9de6a11ce44..fb7e67f2aab 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -97,7 +97,8 @@ public class RankProfileTypeSettingsProcessor extends Processor {
private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) {
for (RankProfile profile : rankProfileRegistry.all()) {
if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence
- profile.addInput(queryFeature, queryFeatureType);
+ profile.addInput(queryFeature,
+ new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty()));
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
index 8cbd94a8a49..edd269559ed 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
@@ -215,7 +215,8 @@ public class ConvertedModel {
for (ImportedMlFunction outputFunction : model.outputExpressions()) {
ExpressionFunction expression = asExpressionFunction(outputFunction);
for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) {
- profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue());
+ Reference name = Reference.fromIdentifier(input.getKey());
+ profile.addInput(name, new RankProfile.Input(name, input.getValue(), Optional.empty()));
}
addExpression(expression, expression.getName(), constantsReplacedByFunctions,
store, profile, queryProfiles, expressions);
@@ -283,7 +284,8 @@ public class ConvertedModel {
String name = output.getFirst();
ExpressionFunction expression = output.getSecond();
for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) {
- profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue());
+ Reference inputName = Reference.fromIdentifier(input.getKey());
+ profile.addInput(inputName, new RankProfile.Input(inputName, input.getValue(), Optional.empty()));
}
TensorType type = expression.getBody().type(profile.typeContext());
if (type != null) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
index d7ce64d1d32..1141af6d79d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -120,7 +120,7 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster
for (var input : rankProfile.inputs().entrySet()) {
var inputConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder();
inputConfig.name(input.getKey().toString());
- inputConfig.type(input.getValue().toString());
+ inputConfig.type(input.getValue().type().toString());
rankProfileConfig.input(inputConfig);
}
docDbBuilder.rankprofile(rankProfileConfig);
diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj
index d2d27a7a2d4..6ce261ebb8c 100644
--- a/config-model/src/main/javacc/IntermediateParser.jj
+++ b/config-model/src/main/javacc/IntermediateParser.jj
@@ -1,9 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// Duplicate Schema parser - work in progress
-// @author arnej27959
-// NOTE: When grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
-
options {
UNICODE_INPUT = true;
CACHE_TOKENS = false;
@@ -43,7 +39,10 @@ import com.yahoo.searchlib.rankingexpression.Reference;
import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
import com.yahoo.searchlib.rankingexpression.evaluation.Value;
import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.IndexedTensor;
+import com.yahoo.tensor.MixedTensor;
import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.TensorAddress;
import java.util.Optional;
import java.util.Map;
@@ -55,6 +54,8 @@ import java.util.logging.Level;
/**
* The schema parser
*
+ * NOTE: When this grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
+ *
* @author bratseth
*/
public class IntermediateParser {
@@ -62,9 +63,7 @@ public class IntermediateParser {
private DeployLogger deployLogger;
private ModelContext.Properties properties;
- /**
- * Creates a parser
- */
+ /** Creates a parser. */
public IntermediateParser(SimpleCharStream stream,
DeployLogger deployLogger,
ModelContext.Properties properties)
@@ -266,7 +265,7 @@ TOKEN :
| < FASTRANK: "fast-rank" >
| < FASTSEARCH: "fast-search" >
| < HUGE: "huge" >
-| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" >
+| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" >
| < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? >
| < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? >
| < COMPRESSION: "compression" >
@@ -303,6 +302,7 @@ TOKEN :
| < ENABLE_BM25: "enable-bm25" >
| < HNSW: "hnsw" >
| < MAXLINKSPERNODE: "max-links-per-node" >
+| < DOUBLE_KEYWORD: "double" >
| < DISTANCEMETRIC: "distance-metric" >
| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" >
| < MULTITHREADEDINDEXING: "multi-threaded-indexing" >
@@ -340,7 +340,8 @@ TOKEN :
| < URI: "uri" >
| < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* >
| < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* >
-| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" >
+| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" >
+| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" >
| < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* >
| < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ >
| < INTEGER: ("-")? (["0"-"9"])+ >
@@ -352,8 +353,8 @@ TOKEN :
| < LESSTHAN: "<" >
| < GREATERTHAN: ">" >
| < VARIABLE: "$" <IDENTIFIER> >
-| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
-| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
+| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
+| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
}
// Declare a special skip token for comments.
@@ -1850,7 +1851,7 @@ void rankProfileItem(ParsedRankProfile profile) : { }
}
/**
- * This rule consumes an inherits statement of a rank-profile.
+ * Consumes an inherits statement of a rank-profile.
*
* @param profile the profile to modify
*/
@@ -2067,10 +2068,35 @@ void inputs(ParsedRankProfile profile) :
}
{
<INPUTS> <LBRACE> (<NL>)*
- ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) *
+ ( input(profile) (<NL>)*) *
<RBRACE>
}
+void input(ParsedRankProfile profile) :
+{
+ Reference reference;
+ TensorType type;
+ Tensor defaultValue = null;
+}
+{
+ reference = queryFeature() type = inputType(reference) ( <COLON> (<NL>)* defaultValue = tensorValue(type) )?
+ { profile.addInput(reference, new RankProfile.Input(reference, type, Optional.ofNullable(defaultValue))); }
+}
+
+TensorType inputType(Reference reference) :
+{
+ TensorType type;
+
+}
+{
+ (
+ ( type = tensorType("Type of " + reference) )
+ |
+ ( <DOUBLE_KEYWORD> { type = TensorType.empty; } )
+ )
+ { return type; }
+}
+
Reference queryFeature() :
{
String argument;
@@ -2243,7 +2269,7 @@ void approximateThreshold(ParsedRankProfile profile) :
}
/**
- * This rule consumes a rank-properties block of a rank profile. There
+ * Consumes a rank-properties block of a rank profile. There
* is a little trick within this rule to allow the final rank property
* to skip the terminating newline token.
*
@@ -2256,7 +2282,7 @@ void rankProperties(ParsedRankProfile profile) : { }
}
/**
- * This rule consumes a single rank property pair for a rank profile.
+ * Consumes a single rank property pair for a rank profile.
*
* @param profile the rank profile to modify
*/
@@ -2270,25 +2296,28 @@ void rankProperty(ParsedRankProfile profile) :
}
/**
- * This rule consumes a single rank property for a rank-properties block.
+ * Consumes a single rank property for a rank-properties block.
*
- * @return The token image of the consumed item.
+ * @return the token image of the consumed item
*/
String rankPropertyItem() :
{
- String image, ret = "";
+ String image = null;
+ String ret = "";
+ Token dToken = null;
}
{
- ( ( image = identifierWithDash() { ret += image; }
+ ( ( image = identifierWithDash() { ret += image; }
+ | dToken = <DOUBLE> { ret += dToken.image; }
| image = quotedString() { ret += image; }
| ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ )
{ return ret; }
}
/**
- * This rule consumes a field-weight statement of a rank profile.
+ * Consumes a field-weight statement of a rank profile.
*
- * @param profile The rank profile to modify.
+ * @param profile the rank profile to modify
*/
void fieldWeight(ParsedRankProfile profile) :
{
@@ -2301,9 +2330,9 @@ void fieldWeight(ParsedRankProfile profile) :
}
/**
- * This rule consumes a rank-type statement of a rank profile.
+ * Consumes a rank-type statement of a rank profile.
*
- * @param profile The rank profile to modify.
+ * @param profile the rank profile to modify
*/
void fieldRankType(ParsedRankProfile profile) :
{
@@ -2316,9 +2345,9 @@ void fieldRankType(ParsedRankProfile profile) :
}
/**
- * This rule consumes a rank filter statement of a rank profile.
+ * Consumes a rank filter statement of a rank profile.
*
- * @param profile The rank profile to modify.
+ * @param profile the rank profile to modify
*/
void fieldRankFilter(ParsedRankProfile profile) :
{
@@ -2330,7 +2359,7 @@ void fieldRankFilter(ParsedRankProfile profile) :
}
/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
+ * Consumes part of a rank-degradation statement of a rank profile.
*/
void rankDegradationBinSize() :
{
@@ -2343,7 +2372,7 @@ void rankDegradationBinSize() :
/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
+ * Consumes part of a rank-degradation statement of a rank profile.
*/
void rankDegradationBinLow() :
{
@@ -2354,9 +2383,8 @@ void rankDegradationBinLow() :
{ deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); }
}
-
/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
+ * Consumes part of a rank-degradation statement of a rank profile.
*/
void rankDegradationPosbinSize() :
{
@@ -2369,7 +2397,7 @@ void rankDegradationPosbinSize() :
/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
+ * Consumes part of a rank-degradation statement of a rank profile.
*/
void rankDegradationItem() : { }
{
@@ -2379,7 +2407,7 @@ void rankDegradationItem() : { }
}
/**
- * This rule consumes a rank-degradation statement of a rank profile.
+ * Consumes a rank-degradation statement of a rank profile.
*/
void rankDegradation() :
{
@@ -2408,10 +2436,10 @@ void constants(ParsedRankProfile profile) :
void constantValue(ParsedRankProfile profile, String name) :
{
- String value;
+ Token value;
}
{
- <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); }
+ <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); }
}
void constantTensor(ParsedRankProfile profile, String name) :
@@ -2421,7 +2449,7 @@ void constantTensor(ParsedRankProfile profile, String name) :
}
{
<LBRACE> (<NL>)*
- (( tensorString = tensorValue() |
+ (( tensorString = tensorValuePrefixedByValue() |
tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE>
{
if (tensorType != null) {
@@ -2437,7 +2465,145 @@ String constantTensorErrorMessage(String rankProfileName, String constantTensorN
{ return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; }
}
-String tensorValue() :
+/**
+ * Parses a tensor written in a tensor literal form,
+ * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form
+ */
+Tensor tensorValue(TensorType type) :
+{
+ Tensor.Builder builder = Tensor.Builder.of(type);
+ Number doubleValue = null;
+}
+{
+ ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = consumeNumber() )
+ {
+ if (doubleValue != null) {
+ if (type.rank() > 0)
+ throw new IllegalArgumentException("A tensor of type " + type + " cannot be a number");
+ builder.cell(doubleValue.doubleValue());
+ }
+ return builder.build();
+ }
+}
+
+/** A mapped or mixed tensor value. */
+void mappedTensorValue(Tensor.Builder builder) : {}
+{
+ "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}"
+}
+
+
+void mappedTensorBlock(Tensor.Builder builder) :
+{
+ TensorAddress mappedAddress;
+}
+{
+ mappedAddress = tensorAddress(builder.type().mappedSubtype()) <COLON> (<NL>)*
+ ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) )
+}
+
+void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builder) :
+{
+ List<Double> values = new ArrayList<Double>();
+}
+{
+ arrayTensorValues(values)
+ {
+ MixedTensor.BoundBuilder boundBuilder = (MixedTensor.BoundBuilder)builder;
+ double[] arrayValues = new double[values.size()];
+ for (int i = 0; i < values.size(); i++ ) {
+ arrayValues[i] = values.get(i);
+ }
+ boundBuilder.block(sparseAddress, arrayValues);
+ }
+}
+
+void indexedTensorValues(Tensor.Builder builder) :
+{
+ List<Double> values = new ArrayList<Double>();
+}
+{
+ arrayTensorValues(values)
+ {
+ IndexedTensor.BoundBuilder boundBuilder = (IndexedTensor.BoundBuilder)builder;
+ double[] arrayValues = new double[values.size()];
+ for (int i = 0; i < values.size(); i++ ) {
+ arrayValues[i] = values.get(i);
+ }
+ boundBuilder.fill(arrayValues);
+ }
+}
+
+/** Tensor array values. Using sub-bracketing for multiple dimensions is optional and therefore ignored here. */
+void arrayTensorValues(List<Double> values) : {}
+{
+ "[" ( ( indexedTensorValue(values) | arrayTensorValues(values)) )*
+ ( <COMMA> (<NL>)* ( indexedTensorValue(values) | arrayTensorValues(values)) )*
+ "]"
+}
+
+void indexedTensorValue(List<Double> values) :
+{
+ Number value;
+}
+{
+ value = consumeNumber()
+ { values.add(value.doubleValue()); }
+}
+
+void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) :
+{
+ double value;
+}
+{
+ value = tensorCellValue()
+ { builder.cell(address, value); }
+}
+
+TensorAddress tensorAddress(TensorType type) :
+{
+ TensorAddress.Builder builder = new TensorAddress.Builder(type);
+ String label;
+}
+{
+ (
+ label = tensorAddressLabel() { builder.add(label); }
+ |
+ ( "{" ( tensorAddressElement(builder) )* ( <COMMA> tensorAddressElement(builder) )* "}" )
+ )
+ { return builder.build(); }
+}
+
+void tensorAddressElement(TensorAddress.Builder builder) :
+{
+ String dimension;
+ String label;
+}
+{
+ dimension = identifier() <COLON> (<NL>)* label = tensorAddressLabel()
+ { builder.add(dimension, label); }
+}
+
+String tensorAddressLabel() :
+{
+ String label;
+}
+{
+ ( label = identifier() | label = quotedString() )
+ { return label; }
+}
+
+double tensorCellValue() :
+{
+ Number value;
+}
+{
+ value = consumeNumber()
+ { return value.doubleValue(); }
+}
+
+/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */
+String tensorValuePrefixedByValue() :
{
String tensor;
}
@@ -2462,7 +2628,7 @@ TensorType tensorType(String errorMessage) :
String tensorTypeString;
}
{
- ( <TENSOR_TYPE> ) { tensorTypeString = token.image; }
+ <TENSOR_TYPE> { tensorTypeString = token.image; }
{
TensorType tensorType;
try {
@@ -2555,7 +2721,7 @@ String identifier() : { }
| <DIRECT>
| <DOCUMENT>
| <DOCUMENTSUMMARY>
- | <DOUBLE>
+ | <DOUBLE_KEYWORD>
| <DYNAMIC>
| <ENABLEBITVECTORS>
| <ENABLEONLYBITVECTOR>
@@ -2683,39 +2849,28 @@ String string() : { }
* unescaping of the content, it simply removes the first and last character of the image. However, the token itself can
* contain anything but a double quote.
*
- * @return The unquoted token image.
+ * @return the unquoted token image
*/
String quotedString() : { }
{
- <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); }
+ ( <DOUBLEQUOTEDSTRING> | <SINGLEQUOTEDSTRING> )
+ { return token.image.substring(1, token.image.length() - 1); }
}
-/**
- * This rule consumes a boolean value.
- *
- * @return The consumed boolean value.
- */
+/** A boolean value. */
Boolean bool() : { }
{
( ( <ON> | <TRUE> ) { return true; } |
( <OFF> | <FALSE> ) { return false; } )
}
-/**
- * This rule consumes an integer token and returns its numeric value.
- *
- * @return The consumed integer value.
- */
+/** Consumes an integer token and returns its numeric value. */
int integer() : { }
{
<INTEGER> { return Integer.parseInt(token.image); }
}
-/**
- * This rule consumes a long or integer token and returns its numeric value.
- *
- * @return The consumed long value.
- */
+/** Consumes a long or integer token and returns its numeric value. */
long consumeLong() : { }
{
( <INTEGER> { return Long.parseLong(token.image); } |
@@ -2723,11 +2878,7 @@ long consumeLong() : { }
)
}
-/**
- * This rule consumes a floating-point token and returns its numeric value.
- *
- * @return The consumed value.
- */
+/** Consumes a floating-point token and returns its numeric value. */
double consumeFloat() : { }
{
<DOUBLE> { return Double.valueOf(token.image); }
@@ -2741,9 +2892,7 @@ Number consumeNumber() :
(num = consumeFloat() | num = consumeLong()) { return num; }
}
-/**
- * This rule consumes an opening brace with leading and trailing newline tokens.
- */
+/** Consumes an opening brace with leading and trailing newline tokens. */
void lbrace() : { }
{
(<NL>)* <LBRACE> (<NL>)*
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 0ff9513885f..de2ce3d6938 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -202,6 +202,7 @@ TOKEN :
| < INDEXING: "indexing" >
| < SUMMARYTO: "summary-to" >
| < DOCUMENTSUMMARY: "document-summary" >
+| <DOUBLE_KEYWORD: "double" >
| < RANKTYPE: "rank-type" >
| < WEIGHT: "weight" >
| < TYPE: "type" >
@@ -296,7 +297,7 @@ TOKEN :
| < PAGED: "paged" >
| < FASTSEARCH: "fast-search" >
| < HUGE: "huge" >
-| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" >
+| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" >
| < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? >
| < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? >
| < COMPRESSION: "compression" >
@@ -2196,15 +2197,36 @@ void secondPhaseItem(RankProfile profile) :
}
/** Consumes an inputs block of a rank profile. */
-void inputs(RankProfile profile) :
+void inputs(RankProfile profile) : {}
+{
+ <INPUTS> <LBRACE> (<NL>)*
+ ( input(profile) (<NL>)*) *
+ <RBRACE>
+}
+
+void input(RankProfile profile) :
{
Reference reference;
TensorType type;
}
{
- <INPUTS> <LBRACE> (<NL>)*
- ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) *
- <RBRACE>
+ reference = queryFeature() type = inputType(reference)
+ {
+ profile.addInput(reference, new RankProfile.Input(reference, type, Optional.empty()));
+ }
+}
+
+TensorType inputType(Reference reference) :
+{
+ TensorType type;
+}
+{
+ (
+ ( type = tensorType("Type of " + reference) )
+ |
+ ( <DOUBLE_KEYWORD> { type = TensorType.empty; } )
+ )
+ { return type; }
}
Reference queryFeature() :
@@ -2408,10 +2430,13 @@ void rankProperty(RankProfile profile) :
*/
String rankPropertyItem() :
{
- String image, ret = "";
+ String image;
+ String ret = "";
+ Token dToken;
}
{
- ( ( image = identifierWithDash() { ret += image; }
+ ( ( image = identifierWithDash() { ret += image; }
+ | dToken = <DOUBLE> { ret += dToken.image; }
| image = quotedString() { ret += image; }
| ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ )
{ return ret; }
@@ -2542,10 +2567,10 @@ void constants(RankProfile profile) :
void constantValue(RankProfile profile, String name) :
{
- String value;
+ Token value;
}
{
- <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); }
+ <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); }
}
void constantTensor(RankProfile profile, String name) :
@@ -2596,7 +2621,7 @@ TensorType tensorType(String errorMessage) :
String tensorTypeString;
}
{
- ( <TENSOR_TYPE> ) { tensorTypeString = token.image; }
+ <TENSOR_TYPE> { tensorTypeString = token.image; }
{
TensorType tensorType;
try {
@@ -2693,7 +2718,7 @@ String identifier() : { }
| <DIRECT>
| <DOCUMENT>
| <DOCUMENTSUMMARY>
- | <DOUBLE>
+ | <DOUBLE_KEYWORD>
| <DYNAMIC>
| <ENABLEBITVECTORS>
| <ENABLEONLYBITVECTOR>
diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd
index 9069f59bbe3..aed80f77e6f 100644
--- a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd
+++ b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd
@@ -77,7 +77,7 @@ search neuralnet {
query(W_1) tensor(hidden[9],out[9])
query(b_1) tensor(out[9])
query(W_out) tensor(out[9])
- query(b_out) tensor(out[1])
+ query(b_out) tensor(out[1]):[1.0]
}
}
diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg
index f5134dd15f9..1e05fe77324 100644
--- a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg
+++ b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg
@@ -11,6 +11,8 @@ rankprofile[].fef.property[].name "vespa.type.query.W_out"
rankprofile[].fef.property[].value "tensor(out[9])"
rankprofile[].fef.property[].name "vespa.type.query.b_out"
rankprofile[].fef.property[].value "tensor(out[1])"
+rankprofile[].fef.property[].name "query(b_out)"
+rankprofile[].fef.property[].value "{{out:0}:1.0}"
rankprofile[].name "unranked"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
rankprofile[].fef.property[].value "value(0)"
@@ -69,18 +71,20 @@ rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScrip
rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
rankprofile[].fef.property[].value "nativeRank"
-rankprofile[].fef.property[].name "vespa.type.query.b_out"
-rankprofile[].fef.property[].value "tensor(out[1])"
-rankprofile[].fef.property[].name "vespa.type.query.W_out"
-rankprofile[].fef.property[].value "tensor(out[9])"
+rankprofile[].fef.property[].name "vespa.type.query.W_0"
+rankprofile[].fef.property[].value "tensor(hidden[9],x[9])"
rankprofile[].fef.property[].name "vespa.type.query.b_0"
rankprofile[].fef.property[].value "tensor(hidden[9])"
-rankprofile[].fef.property[].name "vespa.type.query.b_1"
-rankprofile[].fef.property[].value "tensor(out[9])"
rankprofile[].fef.property[].name "vespa.type.query.W_1"
rankprofile[].fef.property[].value "tensor(hidden[9],out[9])"
-rankprofile[].fef.property[].name "vespa.type.query.W_0"
-rankprofile[].fef.property[].value "tensor(hidden[9],x[9])"
+rankprofile[].fef.property[].name "vespa.type.query.b_1"
+rankprofile[].fef.property[].value "tensor(out[9])"
+rankprofile[].fef.property[].name "vespa.type.query.b_out"
+rankprofile[].fef.property[].value "tensor(out[1])"
+rankprofile[].fef.property[].name "query(b_out)"
+rankprofile[].fef.property[].value "{{out:0}:1.0}"
+rankprofile[].fef.property[].name "vespa.type.query.W_out"
+rankprofile[].fef.property[].value "tensor(out[9])"
rankprofile[].name "neuralNetworkProfile"
rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScript"
rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)"
@@ -172,15 +176,17 @@ rankprofile[].fef.property[].name "vespa.rank.secondphase"
rankprofile[].fef.property[].value "rankingExpression(layer_out)"
rankprofile[].fef.property[].name "vespa.hitcollector.heapsize"
rankprofile[].fef.property[].value "2000"
-rankprofile[].fef.property[].name "vespa.type.query.b_out"
-rankprofile[].fef.property[].value "tensor(out[1])"
-rankprofile[].fef.property[].name "vespa.type.query.W_out"
-rankprofile[].fef.property[].value "tensor(out[9])"
+rankprofile[].fef.property[].name "vespa.type.query.W_0"
+rankprofile[].fef.property[].value "tensor(hidden[9],x[9])"
rankprofile[].fef.property[].name "vespa.type.query.b_0"
rankprofile[].fef.property[].value "tensor(hidden[9])"
-rankprofile[].fef.property[].name "vespa.type.query.b_1"
-rankprofile[].fef.property[].value "tensor(out[9])"
rankprofile[].fef.property[].name "vespa.type.query.W_1"
rankprofile[].fef.property[].value "tensor(hidden[9],out[9])"
-rankprofile[].fef.property[].name "vespa.type.query.W_0"
-rankprofile[].fef.property[].value "tensor(hidden[9],x[9])"
+rankprofile[].fef.property[].name "vespa.type.query.b_1"
+rankprofile[].fef.property[].value "tensor(out[9])"
+rankprofile[].fef.property[].name "vespa.type.query.W_out"
+rankprofile[].fef.property[].value "tensor(out[9])"
+rankprofile[].fef.property[].name "vespa.type.query.b_out"
+rankprofile[].fef.property[].value "tensor(out[1])"
+rankprofile[].fef.property[].name "query(b_out)"
+rankprofile[].fef.property[].value "{{out:0}:1.0}"
diff --git a/config-model/src/test/derived/rankproperties/rank-profiles.cfg b/config-model/src/test/derived/rankproperties/rank-profiles.cfg
index 47a0438a323..3ca44288282 100644
--- a/config-model/src/test/derived/rankproperties/rank-profiles.cfg
+++ b/config-model/src/test/derived/rankproperties/rank-profiles.cfg
@@ -1,6 +1,8 @@
rankprofile[].name "default"
-rankprofile[].fef.property[].name "$test"
+rankprofile[].fef.property[].name "$test1"
rankprofile[].fef.property[].value "foo"
+rankprofile[].fef.property[].name "query(test2)"
+rankprofile[].fef.property[].value "12.3"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
rankprofile[].fef.property[].value "nativeFieldMatch"
rankprofile[].fef.property[].name "vespa.rank.secondphase"
@@ -21,8 +23,10 @@ rankprofile[].fef.property[].value "0"
rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures"
rankprofile[].fef.property[].value "true"
rankprofile[].name "child"
-rankprofile[].fef.property[].name "$test"
+rankprofile[].fef.property[].name "$test1"
rankprofile[].fef.property[].value "foo"
+rankprofile[].fef.property[].name "query(test2)"
+rankprofile[].fef.property[].value "12.3"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
rankprofile[].fef.property[].value "nativeFieldMatch"
rankprofile[].fef.property[].name "vespa.rank.secondphase"
diff --git a/config-model/src/test/derived/rankproperties/rankproperties.sd b/config-model/src/test/derived/rankproperties/rankproperties.sd
index 24977ae645f..db259f1daec 100644
--- a/config-model/src/test/derived/rankproperties/rankproperties.sd
+++ b/config-model/src/test/derived/rankproperties/rankproperties.sd
@@ -28,7 +28,8 @@ search rankproperties {
expression: match
}
rank-properties {
- $test:"foo"
+ $test1:"foo"
+ query(test2): 12.3
#$weight:1
}
}
diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd
index e26358fef5a..339d147dfd0 100644
--- a/config-model/src/test/examples/rankpropvars.sd
+++ b/config-model/src/test/examples/rankpropvars.sd
@@ -55,13 +55,11 @@ document music {
}
field artist type string {
- ## index-to: a
indexing: index | summary
}
field year type int {
indexing: attribute | summary
- ## index-to: y
}
field url type uri {}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java
index 86ccb816c10..0f29d2dda40 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java
@@ -10,8 +10,10 @@ import java.io.IOException;
* @author bratseth
*/
public class RankPropertiesTestCase extends AbstractExportingTestCase {
+
@Test
public void testRankProperties() throws IOException, ParseException {
assertCorrectDeriving("rankproperties");
}
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
index bcb197ed540..0b7dbf2eb21 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
@@ -435,7 +435,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest {
}
private VespaModel getIndexedSearchVespaModel(String xml) {
- List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist");
+ List<String> sds = generateSchemas("music", "album", "artist");
return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
}
@@ -501,4 +501,55 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest {
}
}
+ public static String generateSchema(String name, String field1, String field2) {
+ return "schema " + name + " {" +
+ " document " + name + " {" +
+ " field " + field1 + " type string {\n" +
+ " indexing: index | summary\n" +
+ " summary: dynamic\n" +
+ " }\n" +
+ " field " + field2 + " type int {\n" +
+ " indexing: attribute | summary\n" +
+ " attribute: fast-access\n" +
+ " }\n" +
+ " field " + field2 + "_nfa type int {\n" +
+ " indexing: attribute \n" +
+ " }\n" +
+ " }\n" +
+ " rank-profile staticrank inherits default {" +
+ " first-phase { expression: attribute(" + field2 + ") }" +
+ " }" +
+ " rank-profile summaryfeatures inherits default {" +
+ " first-phase { expression: attribute(" + field2 + ") }\n" +
+ " summary-features: attribute(" + field2 + ")" +
+ " }" +
+ " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" +
+ " }" +
+ " rank-profile rankfeatures {" +
+ " first-phase { expression: attribute(" + field2 + ") }\n" +
+ " rank-features: attribute(" + field2 + ")" +
+ " }" +
+ " rank-profile inputs {" +
+ " inputs {" +
+ " query(foo) tensor<float>(x[10])" +
+ " query(bar) tensor(key{},x[1000])" +
+ " }" +
+ " }" +
+ "}";
+ }
+
+ public static List<String> generateSchemas(String ... sdNames) {
+ return generateSchemas(Arrays.asList(sdNames));
+ }
+
+ public static List<String> generateSchemas(List<String> sdNames) {
+ List<String> sds = new ArrayList<>();
+ int i = 0;
+ for (String sdName : sdNames) {
+ sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2)));
+ i = i + 2;
+ }
+ return sds;
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index 3cff04198bd..671a9f6660d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test;
import com.google.common.collect.ImmutableMap;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.deploy.TestProperties;
-import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.vespa.config.search.IndexschemaConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import com.yahoo.vespa.config.search.RankProfilesConfig;
@@ -16,11 +15,8 @@ import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.content.ContentSearchCluster;
import com.yahoo.vespa.model.content.utils.DocType;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
-import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
-import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
import org.junit.Test;
-import java.util.ArrayList;
import java.util.List;
import java.util.Arrays;
import java.util.Map;
@@ -35,92 +31,9 @@ public class DocumentDatabaseTestCase {
private static final double SMALL = 0.00000000000001;
- private static final String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" +
- "<hosts> " +
- " <host name='foo'>" +
- " <alias>node0</alias>" +
- " </host>" +
- "</hosts>";
-
- private String createVespaServices(List<String> sds, String mode) {
- List<DocType> nameAndModes = new ArrayList<>(sds.size());
- for (String sd : sds) {
- nameAndModes.add(DocType.create(sd, mode));
- }
- return createVespaServicesXml(nameAndModes, "");
- }
- private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) {
- StringBuilder retval = new StringBuilder();
- retval.append("" +
- "<?xml version='1.0' encoding='utf-8' ?>\n" +
- "<services version='1.0'>\n" +
- "<admin version='2.0'>\n" +
- " <adminserver hostalias='node0' />\n" +
- "</admin>\n" +
- "<container version='1.0'>\n" +
- " <nodes>\n" +
- " <node hostalias='node0'/>\n" +
- " </nodes>\n" +
- " <search/>\n" +
- "</container>\n" +
- "<content version='1.0' id='test'>\n" +
- " <redundancy>1</redundancy>\n");
- retval.append(DocType.listToXml(nameAndModes));
- retval.append(
- " <engine>\n" +
- " <proton>\n" +
- " <tuning>\n" +
- " <searchnode>\n" +
- xmlTuning +
- " </searchnode>\n" +
- " </tuning\n>" +
- " </proton\n>" +
- " </engine\n>" +
- " <nodes>\n" +
- " <node hostalias='node0' distribution-key='0'/>\n" +
- " </nodes>\n" +
- " </content>\n" +
- "</services>\n");
- return retval.toString();
- }
-
- private ProtonConfig getProtonCfg(ContentSearchCluster cluster) {
- ProtonConfig.Builder pb = new ProtonConfig.Builder();
- cluster.getConfig(pb);
- return new ProtonConfig(pb);
- }
-
- private void assertSingleSD(String mode) {
- final List<String> sds = Arrays.asList("type1");
- VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
- ApplicationPackageUtils.generateSchemas(sds)).create();
- IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
- ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
- assertEquals(1, indexedSearchCluster.getDocumentDbs().size());
- String type1Id = "test/search/cluster.test/type1";
- ProtonConfig proton = getProtonCfg(contentSearchCluster);
- assertEquals(1, proton.documentdb().size());
- assertEquals("type1", proton.documentdb(0).inputdoctypename());
- assertEquals(type1Id, proton.documentdb(0).configid());
- }
-
@Test
public void requireThatWeCanHaveOneSDForIndexedMode() {
- assertSingleSD("index");
- }
-
- private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) {
- return createModel(nameAndModes, xmlTuning, null);
- }
-
- private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) {
- List<String> sds = new ArrayList<>(nameAndModes.size());
- for (DocType nameAndMode : nameAndModes) {
- sds.add(nameAndMode.getType());
- }
- var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning),
- ApplicationPackageUtils.generateSchemas(sds));
- return builder != null ? creator.create(builder) : creator.create();
+ new SchemaTester().assertSingleSD("index");
}
@Test
@@ -178,9 +91,10 @@ public class DocumentDatabaseTestCase {
if (featureFlagConcurrency != null) {
properties.setFeedConcurrency(featureFlagConcurrency);
}
- VespaModel model = createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties));
+ var tester = new SchemaTester();
+ VespaModel model = tester.createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties));
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
- ProtonConfig proton = getProtonCfg(contentSearchCluster);
+ ProtonConfig proton = tester.getProtonCfg(contentSearchCluster);
assertEquals(global, proton.feeding().concurrency(), SMALL);
assertEquals(local.size(), proton.documentdb().size());
for (int i = 0; i < local.size(); i++) {
@@ -190,11 +104,12 @@ public class DocumentDatabaseTestCase {
@Test
public void requireThatModeIsSet() {
- VespaModel model = createModel(Arrays.asList(DocType.create("a", "index"),
- DocType.create("b", "streaming"),
- DocType.create("c", "store-only")), "");
+ var tester = new SchemaTester();
+ VespaModel model = tester.createModel(Arrays.asList(DocType.create("a", "index"),
+ DocType.create("b", "streaming"),
+ DocType.create("c", "store-only")), "");
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
- ProtonConfig proton = getProtonCfg(contentSearchCluster);
+ ProtonConfig proton = tester.getProtonCfg(contentSearchCluster);
assertEquals(3, proton.documentdb().size());
assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode());
assertEquals("a", proton.documentdb(0).inputdoctypename());
@@ -205,10 +120,11 @@ public class DocumentDatabaseTestCase {
}
private void verifyInitialDocumentCount(List<DocType> nameAndModes, String xmlTuning, List<Long> local) {
+ var tester = new SchemaTester();
assertEquals(nameAndModes.size(), local.size());
- VespaModel model = createModel(nameAndModes, xmlTuning);
+ VespaModel model = tester.createModel(nameAndModes, xmlTuning);
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
- ProtonConfig proton = getProtonCfg(contentSearchCluster);
+ ProtonConfig proton = tester.getProtonCfg(contentSearchCluster);
assertEquals(local.size(), proton.documentdb().size());
for (int i = 0; i < local.size(); i++) {
assertEquals(local.get(i).longValue(), proton.documentdb(i).allocation().initialnumdocs());
@@ -240,14 +156,14 @@ public class DocumentDatabaseTestCase {
assertEquals(attributeField, acfg.attribute(0).name());
assertEquals(attributeField+"_nfa", acfg.attribute(1).name());
RankProfilesConfig rcfg = model.getConfig(RankProfilesConfig.class, configId);
- assertEquals(7, rcfg.rankprofile().size());
+ assertEquals(6, rcfg.rankprofile().size());
}
@Test
public void testMultipleSchemas() {
List<String> sds = List.of("type1", "type2", "type3");
- VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"),
- ApplicationPackageUtils.generateSchemas(sds)).create();
+ var tester = new SchemaTester();
+ var model = tester.createModel(sds);
IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
String type1Id = "test/search/cluster.test/type1";
@@ -255,7 +171,7 @@ public class DocumentDatabaseTestCase {
String type3Id = "test/search/cluster.test/type3";
{
assertEquals(3, indexedSearchCluster.getDocumentDbs().size());
- ProtonConfig proton = getProtonCfg(contentSearchCluster);
+ ProtonConfig proton = tester.getProtonCfg(contentSearchCluster);
assertEquals(3, proton.documentdb().size());
assertEquals("type1", proton.documentdb(0).inputdoctypename());
assertEquals(type1Id, proton.documentdb(0).configid());
@@ -295,9 +211,16 @@ public class DocumentDatabaseTestCase {
@Test
public void requireThatRelevantConfigIsAvailableForClusterSearcher() {
- List<String> schemas = Arrays.asList("type1", "type2");
- VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"),
- ApplicationPackageUtils.generateSchemas(schemas)).create();
+ String inputsProfile =
+ " rank-profile inputs {" +
+ " inputs {" +
+ " query(foo) tensor<float>(x[10])" +
+ " query(bar) tensor(key{},x[1000])" +
+ " }" +
+ " }";
+ List<String> schemas = List.of("type1", "type2");
+ var tester = new SchemaTester();
+ VespaModel model = tester.createModelWithRankProfile(inputsProfile, schemas);
String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher";
{ // documentdb-info config
@@ -309,13 +232,13 @@ public class DocumentDatabaseTestCase {
assertEquals("type1", db.name());
assertEquals(7, db.rankprofile().size());
- assertRankProfile(db, 0, "default", false, false);
- assertRankProfile(db, 1, "unranked", false, false);
- assertRankProfile(db, 2, "staticrank", false, false);
- assertRankProfile(db, 3, "summaryfeatures", true, false);
- assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false);
- assertRankProfile(db, 5, "rankfeatures", false, true);
- var inputs = assertRankProfile(db, 6, "inputs", false, false);
+ tester.assertRankProfile(db, 0, "default", false, false);
+ tester.assertRankProfile(db, 1, "unranked", false, false);
+ tester.assertRankProfile(db, 2, "staticrank", false, false);
+ tester.assertRankProfile(db, 3, "summaryfeatures", true, false);
+ tester.assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false);
+ tester.assertRankProfile(db, 5, "rankfeatures", false, true);
+ var inputs = tester.assertRankProfile(db, 6, "inputs", false, false);
assertEquals(2, inputs.input().size());
assertEquals("query(foo)", inputs.input(0).name());
@@ -326,8 +249,8 @@ public class DocumentDatabaseTestCase {
assertEquals(2, db.summaryclass().size());
assertEquals("default", db.summaryclass(0).name());
assertEquals("attributeprefetch", db.summaryclass(1).name());
- assertSummaryField(db, 0, 0, "f1", "longstring", true);
- assertSummaryField(db, 0, 1, "f2", "integer", false);
+ tester.assertSummaryField(db, 0, 0, "f1", "longstring", true);
+ tester.assertSummaryField(db, 0, 1, "f2", "integer", false);
}
{ // type2
DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(1);
@@ -345,89 +268,10 @@ public class DocumentDatabaseTestCase {
}
}
- /** Schema-info should contain all schemas, independent of clusters. */
- @Test
- public void requireThatSchemaInfoIsAvailable() {
- List<String> schemas = Arrays.asList("type1", "type2");
- VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"),
- ApplicationPackageUtils.generateSchemas(schemas)).create();
- assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model);
- assertSchemaInfo("container", model);
- }
-
- private void assertSchemaInfo(String configId, VespaModel model) {
- { // schema-info config
- SchemaInfoConfig dcfg = model.getConfig(SchemaInfoConfig.class, configId);
- assertEquals(2, dcfg.schema().size());
-
- { // type1
- SchemaInfoConfig.Schema schema = dcfg.schema(0);
- assertEquals("type1", schema.name());
-
- assertEquals(7, schema.rankprofile().size());
- assertRankProfile(schema, 0, "default", false, false);
- assertRankProfile(schema, 1, "unranked", false, false);
- assertRankProfile(schema, 2, "staticrank", false, false);
- assertRankProfile(schema, 3, "summaryfeatures", true, false);
- assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false);
- assertRankProfile(schema, 5, "rankfeatures", false, true);
- var inputs = assertRankProfile(schema, 6, "inputs", false, false);
-
- assertEquals(2, inputs.input().size());
- assertEquals("query(foo)", inputs.input(0).name());
- assertEquals("tensor<float>(x[10])", inputs.input(0).type());
- assertEquals("query(bar)", inputs.input(1).name());
- assertEquals("tensor(key{},x[1000])", inputs.input(1).type());
-
- // assertEquals(2, schema.summaryclass().size());
- // assertEquals("default", schema.summaryclass(0).name());
- // assertEquals("attributeprefetch", schema.summaryclass(1).name());
- // assertSummaryField(schema, 0, 0, "f1", "longstring", true);
- // assertSummaryField(schema, 0, 1, "f2", "integer", false);
- }
- { // type2
- SchemaInfoConfig.Schema schema = dcfg.schema(1);
- assertEquals("type2", schema.name());
- }
- }
- }
-
- private DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db,
- int index,
- String name,
- boolean hasSummaryFeatures,
- boolean hasRankFeatures) {
- DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index);
- assertEquals(name, rankProfile.name());
- assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures());
- assertEquals(hasRankFeatures, rankProfile.hasRankFeatures());
- return rankProfile;
- }
-
- private SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema,
- int index,
- String name,
- boolean hasSummaryFeatures,
- boolean hasRankFeatures) {
- SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index);
- assertEquals(name, rankProfile.name());
- assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures());
- assertEquals(hasRankFeatures, rankProfile.hasRankFeatures());
- return rankProfile;
- }
-
- private void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex,
- String name, String type, boolean dynamic) {
- DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex);
- assertEquals(name, field.name());
- assertEquals(type, field.type());
- assertEquals(dynamic, field.dynamic());
- }
-
private void assertDocumentDBConfigAvailableForStreaming(String mode) {
- final List<String> sds = Arrays.asList("type");
- VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
- ApplicationPackageUtils.generateSchemas(sds)).create();
+ List<String> sds = List.of("type");
+ var tester = new SchemaTester();
+ var model = tester.createModelWithMode(mode, sds);
DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type");
assertEquals(1, dcfg.documentdb().size());
@@ -452,11 +296,11 @@ public class DocumentDatabaseTestCase {
Map<String, List<String>> expectedAttributesMap,
DeployState.Builder builder,
long expectedMaxUnCommittedMemory) {
- VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
- ApplicationPackageUtils.generateSchemas(sds)).create(builder);
+ var tester = new SchemaTester();
+ var model = tester.createModelWithMode(mode, sds, builder);
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
- ProtonConfig proton = getProtonCfg(contentSearchCluster);
+ ProtonConfig proton = tester.getProtonCfg(contentSearchCluster);
assertEquals(sds.size(), proton.documentdb().size());
for (int i = 0; i < sds.size(); i++) {
assertEquals(sds.get(i), proton.documentdb(i).inputdoctypename());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
new file mode 100644
index 00000000000..cf2135aea5a
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
@@ -0,0 +1,106 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search.test;
+
+import com.yahoo.search.config.SchemaInfoConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class SchemaInfoTestCase {
+
+ /** Schema-info should contain all schemas, independent of clusters. */
+ @Test
+ public void requireThatSchemaInfoIsAvailable() {
+ List.of(1.0,2.0,3.0).toArray(new Double[3]);
+ String inputs =
+ " rank-profile inputs {" +
+ " inputs {" +
+ " query(foo) tensor<float>(x[10])" +
+ " query(bar) tensor(key{},x[1000])" +
+ " query(myDouble1) double: 0.5" +
+ " query(myDouble2) tensor()" +
+ " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" +
+ " query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" +
+ " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" +
+ " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" +
+ " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" +
+ " }" +
+ " }";
+ List<String> schemas = List.of("type1", "type2");
+ var tester = new SchemaTester();
+ var model = tester.createModelWithRankProfile(inputs, schemas);
+ assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model, tester);
+ assertSchemaInfo("container", model, tester);
+ }
+
+ private void assertSchemaInfo(String configId, VespaModel model, SchemaTester tester) {
+ {
+ SchemaInfoConfig schemaInfoConfig = model.getConfig(SchemaInfoConfig.class, configId);
+ RankProfilesConfig rankProfilesConfig = model.getConfig(RankProfilesConfig.class, "test/search/cluster.test/type1");
+
+ assertEquals(2, schemaInfoConfig.schema().size());
+
+ { // type1
+ SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(0);
+ assertEquals("type1", schema.name());
+
+ assertEquals(7, schema.rankprofile().size());
+ tester.assertRankProfile(schema, 0, "default", false, false);
+ tester.assertRankProfile(schema, 1, "unranked", false, false);
+ tester.assertRankProfile(schema, 2, "staticrank", false, false);
+ tester.assertRankProfile(schema, 3, "summaryfeatures", true, false);
+ tester.assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false);
+ tester.assertRankProfile(schema, 5, "rankfeatures", false, true);
+
+ var schemaInfoProfile = tester.assertRankProfile(schema, 6, "inputs", false, false);
+ assertEquals(9, schemaInfoProfile.input().size());
+ var rankProfilesProfile = rankProfilesConfig.rankprofile().get(6);
+ assertEquals("inputs", rankProfilesProfile.name());
+ assertInput("query(foo)", "tensor<float>(x[10])", null, 0, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(bar)", "tensor(key{},x[1000])", null, 1, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(myDouble1)", "tensor()", "0.5", 2, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(myDouble2)", "tensor()", null, 3, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(myMap)", "tensor(key{})", "{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(myVector)", "tensor(x[3])", "{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(myMatrix)", "tensor(x[2],y[3])", "{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 6, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(myMixed1)", "tensor(key{},x[2])", "{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 7, schemaInfoProfile, rankProfilesProfile);
+ assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile);
+
+ assertEquals(2, schema.summaryclass().size());
+ assertEquals("default", schema.summaryclass(0).name());
+ assertEquals("attributeprefetch", schema.summaryclass(1).name());
+ tester.assertSummaryField(schema, 0, 0, "f1", "longstring", true);
+ tester.assertSummaryField(schema, 0, 1, "f2", "integer", false);
+ }
+ { // type2
+ SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(1);
+ assertEquals("type2", schema.name());
+ }
+ }
+ }
+
+ private void assertInput(String name, String type, String defaultValue,
+ int index,
+ SchemaInfoConfig.Schema.Rankprofile schemaInfoProfile,
+ RankProfilesConfig.Rankprofile rankProfilesProfile) {
+ assertEquals(name, schemaInfoProfile.input(index).name());
+ assertEquals(type, schemaInfoProfile.input(index).type());
+ if (defaultValue != null) {
+ boolean found = false;
+ for (var property : rankProfilesProfile.fef().property()) {
+ if (property.name().equals(name)) {
+ assertEquals(defaultValue, property.value());
+ found = true;
+ }
+ }
+ if ( ! found)
+ fail("Missing property " + name);
+ }
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java
new file mode 100644
index 00000000000..7a077c1a0aa
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java
@@ -0,0 +1,220 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search.test;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
+import com.yahoo.search.config.SchemaInfoConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.content.ContentSearchCluster;
+import com.yahoo.vespa.model.content.utils.DocType;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class SchemaTester {
+
+ private static final String vespaHosts =
+ "<?xml version='1.0' encoding='utf-8' ?>" +
+ "<hosts> " +
+ " <host name='foo'>" +
+ " <alias>node0</alias>" +
+ " </host>" +
+ "</hosts>";
+
+ private String createVespaServices(List<String> sds, String mode) {
+ List<DocType> nameAndModes = new ArrayList<>(sds.size());
+ for (String sd : sds) {
+ nameAndModes.add(DocType.create(sd, mode));
+ }
+ return createVespaServicesXml(nameAndModes, "");
+ }
+ private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) {
+ StringBuilder retval = new StringBuilder();
+ retval.append("" +
+ "<?xml version='1.0' encoding='utf-8' ?>\n" +
+ "<services version='1.0'>\n" +
+ "<admin version='2.0'>\n" +
+ " <adminserver hostalias='node0' />\n" +
+ "</admin>\n" +
+ "<container version='1.0'>\n" +
+ " <nodes>\n" +
+ " <node hostalias='node0'/>\n" +
+ " </nodes>\n" +
+ " <search/>\n" +
+ "</container>\n" +
+ "<content version='1.0' id='test'>\n" +
+ " <redundancy>1</redundancy>\n");
+ retval.append(DocType.listToXml(nameAndModes));
+ retval.append(
+ " <engine>\n" +
+ " <proton>\n" +
+ " <tuning>\n" +
+ " <searchnode>\n" +
+ xmlTuning +
+ " </searchnode>\n" +
+ " </tuning\n>" +
+ " </proton\n>" +
+ " </engine\n>" +
+ " <nodes>\n" +
+ " <node hostalias='node0' distribution-key='0'/>\n" +
+ " </nodes>\n" +
+ " </content>\n" +
+ "</services>\n");
+ return retval.toString();
+ }
+
+ ProtonConfig getProtonCfg(ContentSearchCluster cluster) {
+ ProtonConfig.Builder pb = new ProtonConfig.Builder();
+ cluster.getConfig(pb);
+ return new ProtonConfig(pb);
+ }
+
+ void assertSingleSD(String mode) {
+ List<String> sds = List.of("type1");
+ VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
+ generateSchemas("", sds)).create();
+ IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
+ ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
+ assertEquals(1, indexedSearchCluster.getDocumentDbs().size());
+ String type1Id = "test/search/cluster.test/type1";
+ ProtonConfig proton = getProtonCfg(contentSearchCluster);
+ assertEquals(1, proton.documentdb().size());
+ assertEquals("type1", proton.documentdb(0).inputdoctypename());
+ assertEquals(type1Id, proton.documentdb(0).configid());
+ }
+
+ VespaModel createModel(List<String> schemas) {
+ return new VespaModelCreatorWithMockPkg(vespaHosts,
+ createVespaServices(schemas, "index"),
+ generateSchemas("", schemas)).create();
+ }
+
+ VespaModel createModelWithRankProfile(String rankProfile, List<String> schemas) {
+ return new VespaModelCreatorWithMockPkg(vespaHosts,
+ createVespaServices(schemas, "index"),
+ generateSchemas(rankProfile, schemas)).create();
+ }
+
+ VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) {
+ return createModel(nameAndModes, xmlTuning, null);
+ }
+
+ VespaModel createModelWithMode(String mode, List<String> schemas) {
+ return new VespaModelCreatorWithMockPkg(vespaHosts,
+ createVespaServices(schemas, mode),
+ generateSchemas("", schemas)).create();
+
+ }
+
+ VespaModel createModelWithMode(String mode, List<String> schemas, DeployState.Builder builder) {
+ return new VespaModelCreatorWithMockPkg(vespaHosts,
+ createVespaServices(schemas, mode),
+ generateSchemas("", schemas)).create(builder);
+ }
+
+ VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) {
+ List<String> sds = new ArrayList<>(nameAndModes.size());
+ for (DocType nameAndMode : nameAndModes) {
+ sds.add(nameAndMode.getType());
+ }
+ var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning),
+ generateSchemas("", sds));
+ return builder != null ? creator.create(builder) : creator.create();
+ }
+
+ public static String generateSchema(String name, String field1, String field2, String rankProfile) {
+ return "schema " + name + " {" +
+ " document " + name + " {" +
+ " field " + field1 + " type string {\n" +
+ " indexing: index | summary\n" +
+ " summary: dynamic\n" +
+ " }\n" +
+ " field " + field2 + " type int {\n" +
+ " indexing: attribute | summary\n" +
+ " attribute: fast-access\n" +
+ " }\n" +
+ " field " + field2 + "_nfa type int {\n" +
+ " indexing: attribute \n" +
+ " }\n" +
+ " }\n" +
+ " rank-profile staticrank inherits default {" +
+ " first-phase { expression: attribute(" + field2 + ") }" +
+ " }" +
+ " rank-profile summaryfeatures inherits default {" +
+ " first-phase { expression: attribute(" + field2 + ") }\n" +
+ " summary-features: attribute(" + field2 + ")" +
+ " }" +
+ " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" +
+ " }" +
+ " rank-profile rankfeatures {" +
+ " first-phase { expression: attribute(" + field2 + ") }\n" +
+ " rank-features: attribute(" + field2 + ")" +
+ " }" +
+ rankProfile +
+ "}";
+ }
+
+ public static List<String> generateSchemas(String rankProfile, String ... sdNames) {
+ return generateSchemas(rankProfile, Arrays.asList(sdNames));
+ }
+
+ public static List<String> generateSchemas(String rankProfile, List<String> sdNames) {
+ List<String> sds = new ArrayList<>();
+ int i = 0;
+ for (String sdName : sdNames) {
+ sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), rankProfile));
+ i = i + 2;
+ }
+ return sds;
+ }
+
+ DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db,
+ int index,
+ String name,
+ boolean hasSummaryFeatures,
+ boolean hasRankFeatures) {
+ DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index);
+ assertEquals(name, rankProfile.name());
+ assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures());
+ assertEquals(hasRankFeatures, rankProfile.hasRankFeatures());
+ return rankProfile;
+ }
+
+ SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema,
+ int index,
+ String name,
+ boolean hasSummaryFeatures,
+ boolean hasRankFeatures) {
+ SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index);
+ assertEquals(name, rankProfile.name());
+ assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures());
+ assertEquals(hasRankFeatures, rankProfile.hasRankFeatures());
+ return rankProfile;
+ }
+
+ void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex,
+ String name, String type, boolean dynamic) {
+ DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex);
+ assertEquals(name, field.name());
+ assertEquals(type, field.type());
+ assertEquals(dynamic, field.dynamic());
+ }
+
+ void assertSummaryField(SchemaInfoConfig.Schema schema, int summaryClassIndex, int fieldIndex,
+ String name, String type, boolean dynamic) {
+ SchemaInfoConfig.Schema.Summaryclass.Fields field = schema.summaryclass(summaryClassIndex).fields(fieldIndex);
+ assertEquals(name, field.name());
+ assertEquals(type, field.type());
+ assertEquals(dynamic, field.dynamic());
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
index 952de2a6f40..df611deb72a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
@@ -7,6 +7,7 @@ import java.util.List;
/**
* For testing purposes only.
+ *
* @author geirst
*/
public class ApplicationPackageUtils {
diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java
index 00c3a1d1aae..95494190734 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java
@@ -47,7 +47,7 @@ public class Coverage {
protected Coverage(long docs, int nodes, boolean full, int resultSets) {
this(docs, docs, nodes, resultSets, full ? FullCoverageDefinition.EXPLICITLY_FULL
- : FullCoverageDefinition.EXPLICITLY_INCOMPLETE);
+ : FullCoverageDefinition.EXPLICITLY_INCOMPLETE);
}
private Coverage(long docs, long active, int nodes, int resultSets, FullCoverageDefinition fullReason) {
@@ -168,9 +168,8 @@ public class Coverage {
}
/**
- * An int between 0 (inclusive) and 100 (inclusive) representing how many
- * percent coverage the result sets this Coverage instance contains information
- * about had.
+ * An int between 0 (inclusive) and 100 (inclusive) representing the
+ * percent coverage of the result sets this instance contains information about.
*/
public int getResultPercentage() {
if (getResultSets() == 0) {
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
index 19e0e441359..abd23c1822d 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
@@ -147,7 +147,7 @@ public class QueryProfileProperties extends Properties {
private String toShortString(Object value) {
if ( ! (value instanceof Tensor)) return value.toString();
- return ((Tensor)value).toShortString();
+ return ((Tensor)value).toAbbreviatedString();
}
private Object convertByType(CompoundName name, Object value, Map<String, String> context) {
diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java
index 5074a520a4e..aa561f58d7a 100644
--- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java
+++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java
@@ -26,7 +26,7 @@ public class Coverage extends com.yahoo.container.handler.Coverage {
/**
* Will set number of documents present in ideal state
*
- * @param soonActive Number of documents active in ideal state
+ * @param soonActive number of documents active in ideal state
* @return self for chaining
*/
@Beta
@@ -35,10 +35,11 @@ public class Coverage extends com.yahoo.container.handler.Coverage {
/**
* Will set the reasons for degraded coverage as reported by vespa backend.
*
- * @param degradedReason Reason for degradation
+ * @param degradedReason reason for degradation
* @return self for chaining
*/
public Coverage setDegradedReason(int degradedReason) { this.degradedReason = degradedReason; return this; }
public Coverage setNodesTried(int nodesTried) { super.setNodesTried(nodesTried); return this; }
+
}
diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json
index 6044666ebf8..f7be61946ba 100644
--- a/vespajlib/abi-spec.json
+++ b/vespajlib/abi-spec.json
@@ -787,7 +787,10 @@
"public",
"abstract"
],
- "methods": [],
+ "methods": [
+ "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(float[])",
+ "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(double[])"
+ ],
"fields": []
},
"com.yahoo.tensor.IndexedTensor$Builder": {
@@ -903,7 +906,8 @@
"public java.util.Map cells()",
"public com.yahoo.tensor.Tensor remove(java.util.Set)",
"public java.lang.String toString()",
- "public java.lang.String toShortString()",
+ "public java.lang.String toString(boolean, boolean)",
+ "public java.lang.String toAbbreviatedString()",
"public boolean equals(java.lang.Object)",
"public bridge synthetic com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)"
],
@@ -954,7 +958,8 @@
"public com.yahoo.tensor.Tensor remove(java.util.Set)",
"public int hashCode()",
"public java.lang.String toString()",
- "public java.lang.String toShortString()",
+ "public java.lang.String toString(boolean, boolean)",
+ "public java.lang.String toAbbreviatedString()",
"public boolean equals(java.lang.Object)"
],
"fields": []
@@ -1046,7 +1051,8 @@
"public com.yahoo.tensor.Tensor remove(java.util.Set)",
"public int hashCode()",
"public java.lang.String toString()",
- "public java.lang.String toShortString()",
+ "public java.lang.String toString(boolean, boolean)",
+ "public java.lang.String toAbbreviatedString()",
"public boolean equals(java.lang.Object)",
"public long denseSubspaceSize()",
"public static com.yahoo.tensor.TensorType createPartialType(com.yahoo.tensor.TensorType$Value, java.util.List)"
@@ -1092,6 +1098,7 @@
],
"methods": [
"public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, java.lang.String)",
+ "public com.yahoo.tensor.TensorType type()",
"public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, long)",
"public com.yahoo.tensor.Tensor$Builder value(double)",
"public com.yahoo.tensor.Tensor$Builder value(float)"
@@ -1233,9 +1240,11 @@
"public java.util.List largest()",
"public java.util.List smallest()",
"public abstract java.lang.String toString()",
- "public abstract java.lang.String toShortString()",
- "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, long)",
- "public static java.lang.String contentToString(com.yahoo.tensor.Tensor, long)",
+ "public abstract java.lang.String toString(boolean, boolean)",
+ "public abstract java.lang.String toAbbreviatedString()",
+ "public java.lang.String toShortString()",
+ "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, boolean, boolean, long)",
+ "public static java.lang.String valueToString(com.yahoo.tensor.Tensor, boolean, long)",
"public abstract boolean equals(java.lang.Object)",
"public abstract int hashCode()",
"public static boolean equals(com.yahoo.tensor.Tensor, com.yahoo.tensor.Tensor)",
@@ -1256,8 +1265,10 @@
],
"methods": [
"public void <init>(com.yahoo.tensor.TensorType)",
+ "public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String)",
"public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String, java.lang.String)",
"public com.yahoo.tensor.TensorAddress$Builder copy()",
+ "public com.yahoo.tensor.TensorType type()",
"public com.yahoo.tensor.TensorAddress build()"
],
"fields": []
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
index 89eefeced56..c4316eb334a 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
@@ -219,21 +219,26 @@ public abstract class IndexedTensor implements Tensor {
}
@Override
- public String toString() { return toString(Long.MAX_VALUE); }
+ public String toString() { return toString(true, true); }
@Override
- public String toShortString() {
- return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1)));
+ public String toString(boolean withType, boolean shortForms) {
+ return toString(withType, shortForms, Long.MAX_VALUE);
}
- private String toString(long maxCells) {
- if (type.rank() == 0) return Tensor.toStandardString(this, maxCells);
- if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty()))
- return Tensor.toStandardString(this, maxCells);
+ @Override
+ public String toAbbreviatedString() {
+ return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1)));
+ }
- Indexes indexes = Indexes.of(dimensionSizes);
+ private String toString(boolean withType, boolean shortForms, long maxCells) {
+ if (! shortForms || type.rank() == 0 || type.dimensions().stream().anyMatch(d -> d.size().isEmpty()))
+ return Tensor.toStandardString(this, withType, shortForms, maxCells);
- StringBuilder b = new StringBuilder(type.toString()).append(":");
+ Indexes indexes = Indexes.of(dimensionSizes);
+ StringBuilder b = new StringBuilder();
+ if (withType)
+ b.append(type).append(":");
indexedBlockToString(this, indexes, maxCells, b);
return b.toString();
}
@@ -438,7 +443,7 @@ public abstract class IndexedTensor implements Tensor {
this.sizes = sizes;
}
- BoundBuilder fill(float[] values) {
+ public BoundBuilder fill(float[] values) {
long index = 0;
for (float value : values) {
cellByDirectIndex(index++, value);
@@ -446,7 +451,7 @@ public abstract class IndexedTensor implements Tensor {
return this;
}
- BoundBuilder fill(double[] values) {
+ public BoundBuilder fill(double[] values) {
long index = 0;
for (double value : values) {
cellByDirectIndex(index++, value);
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java
index ad945ed18bf..946d8fe0f4a 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java
@@ -72,11 +72,18 @@ public class MappedTensor implements Tensor {
public int hashCode() { return cells.hashCode(); }
@Override
- public String toString() { return Tensor.toStandardString(this, Long.MAX_VALUE); }
+ public String toString() { return toString(true, true); }
@Override
- public String toShortString() {
- return Tensor.toStandardString(this, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1)));
+ public String toString(boolean withType, boolean shortForms) { return toString(withType, shortForms, Long.MAX_VALUE); }
+
+ @Override
+ public String toAbbreviatedString() {
+ return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1)));
+ }
+
+ private String toString(boolean withType, boolean shortForms, long maxCells) {
+ return Tensor.toStandardString(this, withType, shortForms, maxCells);
}
@Override
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java
index 56bd94a86e9..d2fed9b96f9 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java
@@ -145,23 +145,27 @@ public class MixedTensor implements Tensor {
@Override
public String toString() {
- return toString(Long.MAX_VALUE);
+ return toString(true, true);
}
@Override
- public String toShortString() {
- return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1)));
+ public String toString(boolean withType, boolean shortForms) {
+ return toString(withType, shortForms, Long.MAX_VALUE);
}
- private String toString(long maxCells) {
- if (type.rank() == 0)
- return Tensor.toStandardString(this, maxCells);
- if (type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty()))
- return Tensor.toStandardString(this, maxCells);
- if (type.dimensions().stream().filter(d -> d.isMapped()).count() > 1)
- return Tensor.toStandardString(this, maxCells);
+ @Override
+ public String toAbbreviatedString() {
+ return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1)));
+ }
+
+ private String toString(boolean withType, boolean shortForms, long maxCells) {
+ if (! shortForms
+ || type.rank() == 0
+ || type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty())
+ || type.dimensions().stream().filter(d -> d.isMapped()).count() > 1)
+ return Tensor.toStandardString(this, withType, shortForms, maxCells);
- return type + ":" + index.contentToString(this, maxCells);
+ return (withType ? type + ":" : "") + index.contentToString(this, maxCells);
}
@Override
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
index 06e7b010a7a..8a84e97fe05 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
@@ -32,7 +32,6 @@ import java.util.Set;
import java.util.function.DoubleBinaryOperator;
import java.util.function.DoubleUnaryOperator;
import java.util.function.Function;
-import java.util.stream.Collectors;
import static com.yahoo.tensor.functions.ScalarFunctions.Hamming;
@@ -316,8 +315,22 @@ public interface Tensor {
@Override
String toString();
+ /**
+ * Returns this tensor on the
+ * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>.
+ *
+ * @param withType whether to prefix the value by the type of this
+ * @param shortForms whether to use short forms where applicable, or always using the verbose form
+ */
+ String toString(boolean withType, boolean shortForms);
+
/** Returns an abbreviated string representation of this tensor suitable for human-readable messages */
- String toShortString();
+ String toAbbreviatedString();
+
+ // TODO: Remove on Vespa 8
+ /** @deprecated use toAbbreviatedString */
+ @Deprecated
+ default String toShortString() { return toAbbreviatedString(); }
/**
* Call this from toString in implementations to return this tensor on the
@@ -325,15 +338,16 @@ public interface Tensor {
* (toString cannot be a default method because default methods cannot override super methods).
*
* @param tensor the tensor to return the standard string format of
+ * @param withType whether the type should be prepended to the content
* @param maxCells the max number of cells to output, after which just , "..." is output to represent the rest
* of the cells
* @return the tensor on the standard string format
*/
- static String toStandardString(Tensor tensor, long maxCells) {
- return tensor.type() + ":" + contentToString(tensor, maxCells);
+ static String toStandardString(Tensor tensor, boolean withType, boolean shortForms, long maxCells) {
+ return (withType ? tensor.type() + ":" : "") + valueToString(tensor, shortForms, maxCells);
}
- static String contentToString(Tensor tensor, long maxCells) {
+ static String valueToString(Tensor tensor, boolean shortForms, long maxCells) {
var cellEntries = new ArrayList<>(tensor.cells().entrySet());
cellEntries.sort(Map.Entry.comparingByKey());
if (tensor.type().dimensions().isEmpty()) {
@@ -345,7 +359,7 @@ public interface Tensor {
for (; i < cellEntries.size() && i < maxCells; i++) {
if (i > 0)
b.append(", ");
- b.append(cellToString(cellEntries.get(i), tensor.type()));
+ b.append(cellToString(cellEntries.get(i), tensor.type(), shortForms));
}
if (i == maxCells && i < tensor.size())
b.append(", ...");
@@ -353,8 +367,9 @@ public interface Tensor {
return b.toString();
}
- private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type) {
- return (type.rank() > 1 ? cell.getKey().toString(type) : TensorAddress.labelToString(cell.getKey().label(0))) +
+ private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type, boolean shortForms) {
+ return (shortForms && type.rank() == 1 ? TensorAddress.labelToString(cell.getKey().label(0))
+ : cell.getKey().toString(type) ) +
":" +
cell.getValue();
}
@@ -533,7 +548,7 @@ public interface Tensor {
return IndexedTensor.Builder.of(type, dimensionSizes);
}
- /** Returns the type this is building */
+ /** Returns the type of the tensor this is building */
TensorType type();
/** Return a cell builder */
@@ -578,6 +593,9 @@ public interface Tensor {
return this;
}
+ /** Returns the type of the tensor this cell is build for. */
+ public TensorType type() { return tensorBuilder.type(); }
+
public CellBuilder label(String dimension, long label) {
return label(dimension, String.valueOf(label));
}
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java
index 27e752f1180..d9ab67d6c5f 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java
@@ -168,7 +168,7 @@ public abstract class TensorAddress implements Comparable<TensorAddress> {
}
- /** Supports building of a tensor address */
+ /** Builder of a tensor address */
public static class Builder {
private final TensorType type;
@@ -184,6 +184,18 @@ public abstract class TensorAddress implements Comparable<TensorAddress> {
}
/**
+ * Adds the label to the only dimension of this.
+ *
+ * @throws IllegalArgumentException if this does not have exactly one dimension
+ */
+ public Builder add(String label) {
+ if (type.rank() != 1)
+ throw new IllegalArgumentException("Cannot add a label without explicit dimension to a tensor of type " + type);
+ add(type.dimensions().get(0).name(), label);
+ return this;
+ }
+
+ /**
* Adds a label in a dimension to this.
*
* @return this for convenience
@@ -203,6 +215,9 @@ public abstract class TensorAddress implements Comparable<TensorAddress> {
return new Builder(type, Arrays.copyOf(labels, labels.length));
}
+ /** Returns the type of the tensor this address is being built for. */
+ public TensorType type() { return type; }
+
public TensorAddress build() {
for (int i = 0; i < labels.length; i++)
if (labels[i] == null)
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
index 2067d7a8492..920f8512c53 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
@@ -65,24 +65,24 @@ public class TensorTestCase {
@Test
public void testToShortString() {
assertEquals("tensor(x[10]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]",
- Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toShortString());
+ Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toAbbreviatedString());
assertEquals("tensor(x[14]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, ...]",
- Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toShortString());
+ Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toAbbreviatedString());
assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0, {d1:l1,d2:l3}:6.0, ...}",
Tensor.from("{{d1:l1,d2:l1}:6, {d2:l2,d1:l1}:6, {d2:l3,d1:l1}:6, {d2:l4,d1:l1}:6, {d2:l5,d1:l1}:6," +
" {d2:l6,d1:l1}:6, {d2:l7,d1:l1}:6, {d2:l8,d1:l1}:6, {d2:l9,d1:l1}:6, {d2:l2,d1:l2}:6," +
- " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toShortString());
+ " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toAbbreviatedString());
assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}",
- Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString());
+ Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString());
assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}",
- Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString());
+ Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString());
assertEquals("tensor(m{},n{},x[3]):{{m:k1,n:k1,x:0}:0.0, {m:k1,n:k1,x:1}:1.0, {m:k1,n:k1,x:2}:2.0, ...}",
Tensor.from("tensor(m{},n{},x[3]):" +
"{{m:k1,n:k1,x:0}:0, {m:k1,n:k1,x:1}:1, {m:k1,n:k1,x:2}:2, " +
" {m:k2,n:k1,x:0}:0, {m:k2,n:k1,x:1}:1, {m:k2,n:k1,x:2}:2, " +
- " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toShortString());
+ " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toAbbreviatedString());
assertEquals("tensor(m{},x[2],y[2]):{k1:[[0.0, 1.0], [2.0, 3.0]], k2:[[0.0, ...}",
- Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toShortString());
+ Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toAbbreviatedString());
}
@Test