diff options
58 files changed, 1008 insertions, 512 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index d091abc3781..9c33045e921 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -113,6 +113,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean experimentalSdParsing() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableBitVectors() { return false; } @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } + @ModelFeatureFlag(owners = {"tokle"}) default boolean enableProxyProtocolMixedMode() { return true; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index e134b8f53ac..11c55521100 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -8,7 +8,7 @@ import com.yahoo.path.Path; import java.nio.ByteBuffer; import java.util.Objects; -public class DistributableResource { +public class DistributableResource implements Comparable <DistributableResource> { public enum PathType { FILE, URI, BLOB } @@ -87,4 +87,8 @@ public class DistributableResource { return "resource '" + name + " of type '" + pathType + "' with ref '" + fileReference + "'"; } + @Override + public int compareTo(DistributableResource o) { + return name.compareTo(o.getName()); + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java index a1299c12307..5e3ef39f360 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -2,10 +2,11 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.FileRegistry; - +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class LargeRankExpressions { private final Map<String, RankExpressionBody> expressions = new ConcurrentHashMap<>(); @@ -36,6 +37,12 @@ public class LargeRankExpressions { } public int limit() { return limit; } + /** Returns a read-only list of ranking constants ordered by name */ + public Collection<RankExpressionBody> expressions() { + return expressions.values().stream().sorted().collect(Collectors.toUnmodifiableList()); + } + + // Note: Use by integration tests in internal repo /** Returns a read-only map of the ranking constants in this indexed by name */ public Map<String, RankExpressionBody> asMap() { return Collections.unmodifiableMap(expressions); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 9c802075462..e1fe795d2b1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -24,9 +24,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.Arguments; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import java.io.File; import java.io.IOException; import java.io.Reader; import java.io.Serializable; @@ -116,7 +116,7 @@ public class RankProfile implements Cloneable { // This cache must be invalidated every time modifications are done to 'functions'. private CachedFunctions allFunctionsCached = null; - private Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private Map<Reference, Input> inputs = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -764,33 +764,31 @@ public class RankProfile implements Cloneable { * All inputs must either be declared through this or in query profile types, * otherwise they are assumes to be scalars. */ - public void addInput(Reference reference, TensorType declaredType) { + public void addInput(Reference reference, Input input) { if (inputs.containsKey(reference)) { - TensorType hadType = inputs().get(reference); - if (! declaredType.equals(hadType)) - throw new IllegalArgumentException("Duplicate input '" + name + "' declared with both type " + - hadType + " and " + declaredType); + Input existing = inputs().get(reference); + if (! input.equals(existing)) + throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing"); } - inputs.put(reference, declaredType); + inputs.put(reference, input); } /** Returns the inputs of this, which also includes all inputs of the parents of this. */ // This is less restrictive than most other constructs in allowing inputs to be defined in all parent profiles // because inputs are tied closer to functions than the profile itself. - public Map<Reference, TensorType> inputs() { + public Map<Reference, Input> inputs() { if (inputs.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(inputs); // Combine - Map<Reference, TensorType> allInputs = new LinkedHashMap<>(); + Map<Reference, Input> allInputs = new LinkedHashMap<>(); for (var inheritedProfile : inherited()) { for (var input : inheritedProfile.inputs().entrySet()) { - TensorType existingType = allInputs.get(input.getKey()); - if (existingType != null && ! existingType.equals(input.getValue())) + Input existing = allInputs.get(input.getKey()); + if (existing != null && ! existing.equals(input.getValue())) throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + - input.getValue() + ", with type " + input.getValue() + "" + - " but this input is already defined with type " + existingType + - " in another profile this inherits"); + input.getValue() + ", but this input is already defined as " + + existing + " in another profile this inherits"); allInputs.put(input.getKey(), input.getValue()); } } @@ -1050,7 +1048,8 @@ public class RankProfile implements Cloneable { public MapEvaluationTypeContext typeContext() { return typeContext(new QueryProfileRegistry()); } private Map<Reference, TensorType> featureTypes() { - Map<Reference, TensorType> featureTypes = new HashMap<>(inputs()); + Map<Reference, TensorType> featureTypes = inputs().values().stream() + .collect(Collectors.toMap(input -> input.name(), input -> input.type())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); return featureTypes; @@ -1070,7 +1069,7 @@ public class RankProfile implements Cloneable { TensorType type = field.getType().asTensorType(); Optional<Reference> feature = Reference.simple(field.getName()); if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue; - if (featureTypes.containsKey(feature)) continue; // Explicit feature types (from inputs) overrides + if (featureTypes.containsKey(feature.get())) continue; // Explicit feature types (from inputs) overrides TensorType existingType = context.getType(feature.get()); if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) @@ -1392,6 +1391,46 @@ public class RankProfile implements Cloneable { } + public static final class Input { + + private final Reference name; + private final TensorType type; + private final Optional<Tensor> defaultValue; + + public Input(Reference name, TensorType type, Optional<Tensor> defaultValue) { + this.name = name; + this.type = type; + this.defaultValue = defaultValue; + } + + public Reference name() { return name; } + public TensorType type() { return type; } + public Optional<Tensor> defaultValue() { return defaultValue; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Input)) return false; + Input other = (Input)o; + if ( ! other.name().equals(this.name())) return false; + if ( ! other.type().equals(this.type())) return false; + if ( ! other.defaultValue().equals(this.defaultValue())) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, defaultValue); + } + + @Override + public String toString() { + return "input '" + name + "' " + type + + (defaultValue().isPresent() ? ":" + defaultValue.get().toAbbreviatedString() : ""); + } + + } + private static class CachedFunctions { private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index cd94b4a7f6e..4c9e65f16c8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -153,7 +153,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } public void getConfig(RankingExpressionsConfig.Builder builder) { - largeRankExpressions.asMap().values().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + largeRankExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); } public void getConfig(RankingConstantsConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 3c14a2b9c63..77245da5ddd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -145,7 +145,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); private final Map<String, String> attributeTypes; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; private final Set<String> filterFields = new java.util.LinkedHashSet<>(); private final String rankprofileName; @@ -426,10 +426,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } - for (Map.Entry<Reference, TensorType> input : inputs.entrySet()) { - if (FeatureNames.isQueryFeature(input.getKey())) - properties.add(new Pair<>("vespa.type.query." + input.getKey().arguments().expressions().get(0), - input.getValue().toString())); + for (var input : inputs.values()) { + if (FeatureNames.isQueryFeature(input.name())) { + properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), + input.type().toString())); + if (input.defaultValue().isPresent()) + properties.add(new Pair<>(input.name().toString(), + input.type().rank() == 0 ? + String.valueOf(input.defaultValue().get().asDouble()) : + input.defaultValue().get().toString(false, false))); + } } if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); distributeLargeExpressionsAsFiles(properties, largeRankExpressions); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java index eeb3a97eda9..0c1e5a76a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -15,6 +16,7 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; /** * Information about a schema. @@ -100,7 +102,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } schemaBuilder.rankprofile(rankProfileConfig); @@ -113,7 +115,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { this.name = profile.name(); @@ -125,7 +127,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } - public Map<Reference, TensorType> inputs() { return inputs; } + public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 118945369d3..63a120f7c7b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; +import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; import com.yahoo.searchdefinition.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; @@ -52,7 +53,7 @@ class ParsedRankProfile extends ParsedBlock { private final Map<String, String> fieldsRankType = new LinkedHashMap<>(); private final Map<String, List<String>> rankProperties = new LinkedHashMap<>(); private final Map<String, Value> constants = new LinkedHashMap<>(); - private final Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private final Map<Reference, RankProfile.Input> inputs = new LinkedHashMap<>(); ParsedRankProfile(String name) { super(name, "rank-profile"); @@ -83,7 +84,7 @@ class ParsedRankProfile extends ParsedBlock { Map<String, String> getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } Map<String, List<String>> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } Map<String, Value> getConstants() { return Collections.unmodifiableMap(constants); } - Map<Reference, TensorType> getInputs() { return Collections.unmodifiableMap(inputs); } + Map<Reference, RankProfile.Input> getInputs() { return Collections.unmodifiableMap(inputs); } Optional<String> getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } @@ -110,9 +111,9 @@ class ParsedRankProfile extends ParsedBlock { constants.put(name, value); } - void addInput(Reference name, TensorType type) { + void addInput(Reference name, RankProfile.Input input) { verifyThat(! inputs.containsKey(name), "already has input", name); - inputs.put(name, type); + inputs.put(name, input); } void addFieldRankFilter(String field, boolean filter) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 9de6a11ce44..fb7e67f2aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -97,7 +97,8 @@ public class RankProfileTypeSettingsProcessor extends Processor { private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) { for (RankProfile profile : rankProfileRegistry.all()) { if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence - profile.addInput(queryFeature, queryFeatureType); + profile.addInput(queryFeature, + new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index ed943317a1f..b9b7f122d63 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -153,8 +153,9 @@ public class RankSetupValidator extends Validator { // Assist verify-ranksetup in finding the actual ONNX model files writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().onnxModels().asMap().values()); - writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().asMap().values()); + writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().expressions()); + config.sort(String::compareTo); String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config); IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index e3aa5bd517f..f55fb547bb0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -25,16 +25,17 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final boolean enforceClientAuth; private final boolean enforceHandshakeClientAuth; private final Collection<String> tlsCiphersOverride; + private final boolean enableProxyProtocolMixedMode; /** * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ public static HostedSslConnectorFactory withProvidedCertificate( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth, - Collection<String> tlsCiphersOverride) { + Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth); - return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride); + return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, enableProxyProtocolMixedMode); } /** @@ -42,25 +43,28 @@ public class HostedSslConnectorFactory extends ConnectorFactory { */ public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates, - Collection<String> tlsCiphersOverride) { + Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false); - return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride); + return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode); } /** * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration). */ - public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, Collection<String> tlsCiphersOverride) { - return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride); + public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, Collection<String> tlsCiphersOverride, + boolean enableProxyProtocolMixedMode) { + return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride, enableProxyProtocolMixedMode); } private HostedSslConnectorFactory(SslProvider sslProvider, boolean enforceClientAuth, - boolean enforceHandshakeClientAuth, Collection<String> tlsCiphersOverride) { + boolean enforceHandshakeClientAuth, Collection<String> tlsCiphersOverride, + boolean enableProxyProtocolMixedMode) { super(new Builder("tls4443", 4443).sslProvider(sslProvider)); this.enforceClientAuth = enforceClientAuth; this.enforceHandshakeClientAuth = enforceHandshakeClientAuth; this.tlsCiphersOverride = tlsCiphersOverride; + this.enableProxyProtocolMixedMode = enableProxyProtocolMixedMode; } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( @@ -94,7 +98,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { } connectorBuilder - .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(true)) + .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(enableProxyProtocolMixedMode)) .idleTimeout(Duration.ofSeconds(30).toSeconds()) .maxConnectionLife(Duration.ofSeconds(45).toSeconds()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index ff1a4b6cc5f..170e22b6164 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -465,6 +465,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // If the deployment contains certificate/private key reference, setup TLS port HostedSslConnectorFactory connectorFactory; Collection<String> tlsCiphersOverride = deployState.getProperties().tlsCiphersOverride(); + boolean proxyProtocolMixedMode = deployState.getProperties().featureFlags().enableProxyProtocolMixedMode(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { @@ -480,11 +481,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore( - serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState), tlsCiphersOverride) + serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState), tlsCiphersOverride, proxyProtocolMixedMode) : HostedSslConnectorFactory.withProvidedCertificate( - serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride); + serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode); } else { - connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride); + connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride, proxyProtocolMixedMode); } cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); server.addConnector(connectorFactory); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 8cbd94a8a49..edd269559ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -215,7 +215,8 @@ public class ConvertedModel { for (ImportedMlFunction outputFunction : model.outputExpressions()) { ExpressionFunction expression = asExpressionFunction(outputFunction); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference name = Reference.fromIdentifier(input.getKey()); + profile.addInput(name, new RankProfile.Input(name, input.getValue(), Optional.empty())); } addExpression(expression, expression.getName(), constantsReplacedByFunctions, store, profile, queryProfiles, expressions); @@ -283,7 +284,8 @@ public class ConvertedModel { String name = output.getFirst(); ExpressionFunction expression = output.getSecond(); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference inputName = Reference.fromIdentifier(input.getKey()); + profile.addInput(inputName, new RankProfile.Input(inputName, input.getValue(), Optional.empty())); } TensorType type = expression.getBody().type(profile.typeContext()); if (type != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index d7ce64d1d32..1141af6d79d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -120,7 +120,7 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } docDbBuilder.rankprofile(rankProfileConfig); diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index d2d27a7a2d4..6ce261ebb8c 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1,9 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Duplicate Schema parser - work in progress -// @author arnej27959 -// NOTE: When grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf - options { UNICODE_INPUT = true; CACHE_TOKENS = false; @@ -43,7 +39,10 @@ import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorAddress; import java.util.Optional; import java.util.Map; @@ -55,6 +54,8 @@ import java.util.logging.Level; /** * The schema parser * + * NOTE: When this grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf + * * @author bratseth */ public class IntermediateParser { @@ -62,9 +63,7 @@ public class IntermediateParser { private DeployLogger deployLogger; private ModelContext.Properties properties; - /** - * Creates a parser - */ + /** Creates a parser. */ public IntermediateParser(SimpleCharStream stream, DeployLogger deployLogger, ModelContext.Properties properties) @@ -266,7 +265,7 @@ TOKEN : | < FASTRANK: "fast-rank" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > -| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < COMPRESSION: "compression" > @@ -303,6 +302,7 @@ TOKEN : | < ENABLE_BM25: "enable-bm25" > | < HNSW: "hnsw" > | < MAXLINKSPERNODE: "max-links-per-node" > +| < DOUBLE_KEYWORD: "double" > | < DISTANCEMETRIC: "distance-metric" > | < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < MULTITHREADEDINDEXING: "multi-threaded-indexing" > @@ -340,7 +340,8 @@ TOKEN : | < URI: "uri" > | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* > | < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* > -| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" > | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > | < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ > | < INTEGER: ("-")? (["0"-"9"])+ > @@ -352,8 +353,8 @@ TOKEN : | < LESSTHAN: "<" > | < GREATERTHAN: ">" > | < VARIABLE: "$" <IDENTIFIER> > -| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > -| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > } // Declare a special skip token for comments. @@ -1850,7 +1851,7 @@ void rankProfileItem(ParsedRankProfile profile) : { } } /** - * This rule consumes an inherits statement of a rank-profile. + * Consumes an inherits statement of a rank-profile. * * @param profile the profile to modify */ @@ -2067,10 +2068,35 @@ void inputs(ParsedRankProfile profile) : } { <INPUTS> <LBRACE> (<NL>)* - ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) * + ( input(profile) (<NL>)*) * <RBRACE> } +void input(ParsedRankProfile profile) : +{ + Reference reference; + TensorType type; + Tensor defaultValue = null; +} +{ + reference = queryFeature() type = inputType(reference) ( <COLON> (<NL>)* defaultValue = tensorValue(type) )? + { profile.addInput(reference, new RankProfile.Input(reference, type, Optional.ofNullable(defaultValue))); } +} + +TensorType inputType(Reference reference) : +{ + TensorType type; + +} +{ + ( + ( type = tensorType("Type of " + reference) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } +} + Reference queryFeature() : { String argument; @@ -2243,7 +2269,7 @@ void approximateThreshold(ParsedRankProfile profile) : } /** - * This rule consumes a rank-properties block of a rank profile. There + * Consumes a rank-properties block of a rank profile. There * is a little trick within this rule to allow the final rank property * to skip the terminating newline token. * @@ -2256,7 +2282,7 @@ void rankProperties(ParsedRankProfile profile) : { } } /** - * This rule consumes a single rank property pair for a rank profile. + * Consumes a single rank property pair for a rank profile. * * @param profile the rank profile to modify */ @@ -2270,25 +2296,28 @@ void rankProperty(ParsedRankProfile profile) : } /** - * This rule consumes a single rank property for a rank-properties block. + * Consumes a single rank property for a rank-properties block. * - * @return The token image of the consumed item. + * @return the token image of the consumed item */ String rankPropertyItem() : { - String image, ret = ""; + String image = null; + String ret = ""; + Token dToken = null; } { - ( ( image = identifierWithDash() { ret += image; } + ( ( image = identifierWithDash() { ret += image; } + | dToken = <DOUBLE> { ret += dToken.image; } | image = quotedString() { ret += image; } | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ ) { return ret; } } /** - * This rule consumes a field-weight statement of a rank profile. + * Consumes a field-weight statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldWeight(ParsedRankProfile profile) : { @@ -2301,9 +2330,9 @@ void fieldWeight(ParsedRankProfile profile) : } /** - * This rule consumes a rank-type statement of a rank profile. + * Consumes a rank-type statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankType(ParsedRankProfile profile) : { @@ -2316,9 +2345,9 @@ void fieldRankType(ParsedRankProfile profile) : } /** - * This rule consumes a rank filter statement of a rank profile. + * Consumes a rank filter statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankFilter(ParsedRankProfile profile) : { @@ -2330,7 +2359,7 @@ void fieldRankFilter(ParsedRankProfile profile) : } /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinSize() : { @@ -2343,7 +2372,7 @@ void rankDegradationBinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinLow() : { @@ -2354,9 +2383,8 @@ void rankDegradationBinLow() : { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); } } - /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationPosbinSize() : { @@ -2369,7 +2397,7 @@ void rankDegradationPosbinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationItem() : { } { @@ -2379,7 +2407,7 @@ void rankDegradationItem() : { } } /** - * This rule consumes a rank-degradation statement of a rank profile. + * Consumes a rank-degradation statement of a rank profile. */ void rankDegradation() : { @@ -2408,10 +2436,10 @@ void constants(ParsedRankProfile profile) : void constantValue(ParsedRankProfile profile, String name) : { - String value; + Token value; } { - <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); } } void constantTensor(ParsedRankProfile profile, String name) : @@ -2421,7 +2449,7 @@ void constantTensor(ParsedRankProfile profile, String name) : } { <LBRACE> (<NL>)* - (( tensorString = tensorValue() | + (( tensorString = tensorValuePrefixedByValue() | tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE> { if (tensorType != null) { @@ -2437,7 +2465,145 @@ String constantTensorErrorMessage(String rankProfileName, String constantTensorN { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; } } -String tensorValue() : +/** + * Parses a tensor written in a tensor literal form, + * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form + */ +Tensor tensorValue(TensorType type) : +{ + Tensor.Builder builder = Tensor.Builder.of(type); + Number doubleValue = null; +} +{ + ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = consumeNumber() ) + { + if (doubleValue != null) { + if (type.rank() > 0) + throw new IllegalArgumentException("A tensor of type " + type + " cannot be a number"); + builder.cell(doubleValue.doubleValue()); + } + return builder.build(); + } +} + +/** A mapped or mixed tensor value. */ +void mappedTensorValue(Tensor.Builder builder) : {} +{ + "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}" +} + + +void mappedTensorBlock(Tensor.Builder builder) : +{ + TensorAddress mappedAddress; +} +{ + mappedAddress = tensorAddress(builder.type().mappedSubtype()) <COLON> (<NL>)* + ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) ) +} + +void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + MixedTensor.BoundBuilder boundBuilder = (MixedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.block(sparseAddress, arrayValues); + } +} + +void indexedTensorValues(Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + IndexedTensor.BoundBuilder boundBuilder = (IndexedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.fill(arrayValues); + } +} + +/** Tensor array values. Using sub-bracketing for multiple dimensions is optional and therefore ignored here. */ +void arrayTensorValues(List<Double> values) : {} +{ + "[" ( ( indexedTensorValue(values) | arrayTensorValues(values)) )* + ( <COMMA> (<NL>)* ( indexedTensorValue(values) | arrayTensorValues(values)) )* + "]" +} + +void indexedTensorValue(List<Double> values) : +{ + Number value; +} +{ + value = consumeNumber() + { values.add(value.doubleValue()); } +} + +void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) : +{ + double value; +} +{ + value = tensorCellValue() + { builder.cell(address, value); } +} + +TensorAddress tensorAddress(TensorType type) : +{ + TensorAddress.Builder builder = new TensorAddress.Builder(type); + String label; +} +{ + ( + label = tensorAddressLabel() { builder.add(label); } + | + ( "{" ( tensorAddressElement(builder) )* ( <COMMA> tensorAddressElement(builder) )* "}" ) + ) + { return builder.build(); } +} + +void tensorAddressElement(TensorAddress.Builder builder) : +{ + String dimension; + String label; +} +{ + dimension = identifier() <COLON> (<NL>)* label = tensorAddressLabel() + { builder.add(dimension, label); } +} + +String tensorAddressLabel() : +{ + String label; +} +{ + ( label = identifier() | label = quotedString() ) + { return label; } +} + +double tensorCellValue() : +{ + Number value; +} +{ + value = consumeNumber() + { return value.doubleValue(); } +} + +/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */ +String tensorValuePrefixedByValue() : { String tensor; } @@ -2462,7 +2628,7 @@ TensorType tensorType(String errorMessage) : String tensorTypeString; } { - ( <TENSOR_TYPE> ) { tensorTypeString = token.image; } + <TENSOR_TYPE> { tensorTypeString = token.image; } { TensorType tensorType; try { @@ -2555,7 +2721,7 @@ String identifier() : { } | <DIRECT> | <DOCUMENT> | <DOCUMENTSUMMARY> - | <DOUBLE> + | <DOUBLE_KEYWORD> | <DYNAMIC> | <ENABLEBITVECTORS> | <ENABLEONLYBITVECTOR> @@ -2683,39 +2849,28 @@ String string() : { } * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can * contain anything but a double quote. * - * @return The unquoted token image. + * @return the unquoted token image */ String quotedString() : { } { - <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); } + ( <DOUBLEQUOTEDSTRING> | <SINGLEQUOTEDSTRING> ) + { return token.image.substring(1, token.image.length() - 1); } } -/** - * This rule consumes a boolean value. - * - * @return The consumed boolean value. - */ +/** A boolean value. */ Boolean bool() : { } { ( ( <ON> | <TRUE> ) { return true; } | ( <OFF> | <FALSE> ) { return false; } ) } -/** - * This rule consumes an integer token and returns its numeric value. - * - * @return The consumed integer value. - */ +/** Consumes an integer token and returns its numeric value. */ int integer() : { } { <INTEGER> { return Integer.parseInt(token.image); } } -/** - * This rule consumes a long or integer token and returns its numeric value. - * - * @return The consumed long value. - */ +/** Consumes a long or integer token and returns its numeric value. */ long consumeLong() : { } { ( <INTEGER> { return Long.parseLong(token.image); } | @@ -2723,11 +2878,7 @@ long consumeLong() : { } ) } -/** - * This rule consumes a floating-point token and returns its numeric value. - * - * @return The consumed value. - */ +/** Consumes a floating-point token and returns its numeric value. */ double consumeFloat() : { } { <DOUBLE> { return Double.valueOf(token.image); } @@ -2741,9 +2892,7 @@ Number consumeNumber() : (num = consumeFloat() | num = consumeLong()) { return num; } } -/** - * This rule consumes an opening brace with leading and trailing newline tokens. - */ +/** Consumes an opening brace with leading and trailing newline tokens. */ void lbrace() : { } { (<NL>)* <LBRACE> (<NL>)* diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 0ff9513885f..de2ce3d6938 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -202,6 +202,7 @@ TOKEN : | < INDEXING: "indexing" > | < SUMMARYTO: "summary-to" > | < DOCUMENTSUMMARY: "document-summary" > +| <DOUBLE_KEYWORD: "double" > | < RANKTYPE: "rank-type" > | < WEIGHT: "weight" > | < TYPE: "type" > @@ -296,7 +297,7 @@ TOKEN : | < PAGED: "paged" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > -| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < COMPRESSION: "compression" > @@ -2196,15 +2197,36 @@ void secondPhaseItem(RankProfile profile) : } /** Consumes an inputs block of a rank profile. */ -void inputs(RankProfile profile) : +void inputs(RankProfile profile) : {} +{ + <INPUTS> <LBRACE> (<NL>)* + ( input(profile) (<NL>)*) * + <RBRACE> +} + +void input(RankProfile profile) : { Reference reference; TensorType type; } { - <INPUTS> <LBRACE> (<NL>)* - ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) * - <RBRACE> + reference = queryFeature() type = inputType(reference) + { + profile.addInput(reference, new RankProfile.Input(reference, type, Optional.empty())); + } +} + +TensorType inputType(Reference reference) : +{ + TensorType type; +} +{ + ( + ( type = tensorType("Type of " + reference) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } } Reference queryFeature() : @@ -2408,10 +2430,13 @@ void rankProperty(RankProfile profile) : */ String rankPropertyItem() : { - String image, ret = ""; + String image; + String ret = ""; + Token dToken; } { - ( ( image = identifierWithDash() { ret += image; } + ( ( image = identifierWithDash() { ret += image; } + | dToken = <DOUBLE> { ret += dToken.image; } | image = quotedString() { ret += image; } | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ ) { return ret; } @@ -2542,10 +2567,10 @@ void constants(RankProfile profile) : void constantValue(RankProfile profile, String name) : { - String value; + Token value; } { - <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); } } void constantTensor(RankProfile profile, String name) : @@ -2596,7 +2621,7 @@ TensorType tensorType(String errorMessage) : String tensorTypeString; } { - ( <TENSOR_TYPE> ) { tensorTypeString = token.image; } + <TENSOR_TYPE> { tensorTypeString = token.image; } { TensorType tensorType; try { @@ -2693,7 +2718,7 @@ String identifier() : { } | <DIRECT> | <DOCUMENT> | <DOCUMENTSUMMARY> - | <DOUBLE> + | <DOUBLE_KEYWORD> | <DYNAMIC> | <ENABLEBITVECTORS> | <ENABLEONLYBITVECTOR> diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd index 9069f59bbe3..aed80f77e6f 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd @@ -77,7 +77,7 @@ search neuralnet { query(W_1) tensor(hidden[9],out[9]) query(b_1) tensor(out[9]) query(W_out) tensor(out[9]) - query(b_out) tensor(out[1]) + query(b_out) tensor(out[1]):[1.0] } } diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg index f5134dd15f9..1e05fe77324 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg @@ -11,6 +11,8 @@ rankprofile[].fef.property[].name "vespa.type.query.W_out" rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.b_out" rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" rankprofile[].name "unranked" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "value(0)" @@ -69,18 +71,20 @@ rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScrip rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeRank" -rankprofile[].fef.property[].name "vespa.type.query.b_out" -rankprofile[].fef.property[].value "tensor(out[1])" -rankprofile[].fef.property[].name "vespa.type.query.W_out" -rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_0" +rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" rankprofile[].fef.property[].name "vespa.type.query.b_0" rankprofile[].fef.property[].value "tensor(hidden[9])" -rankprofile[].fef.property[].name "vespa.type.query.b_1" -rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.W_1" rankprofile[].fef.property[].value "tensor(hidden[9],out[9])" -rankprofile[].fef.property[].name "vespa.type.query.W_0" -rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_1" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_out" +rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" +rankprofile[].fef.property[].name "vespa.type.query.W_out" +rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].name "neuralNetworkProfile" rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScript" rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)" @@ -172,15 +176,17 @@ rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(layer_out)" rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" rankprofile[].fef.property[].value "2000" -rankprofile[].fef.property[].name "vespa.type.query.b_out" -rankprofile[].fef.property[].value "tensor(out[1])" -rankprofile[].fef.property[].name "vespa.type.query.W_out" -rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_0" +rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" rankprofile[].fef.property[].name "vespa.type.query.b_0" rankprofile[].fef.property[].value "tensor(hidden[9])" -rankprofile[].fef.property[].name "vespa.type.query.b_1" -rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.W_1" rankprofile[].fef.property[].value "tensor(hidden[9],out[9])" -rankprofile[].fef.property[].name "vespa.type.query.W_0" -rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_1" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_out" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_out" +rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" diff --git a/config-model/src/test/derived/rankproperties/rank-profiles.cfg b/config-model/src/test/derived/rankproperties/rank-profiles.cfg index 47a0438a323..3ca44288282 100644 --- a/config-model/src/test/derived/rankproperties/rank-profiles.cfg +++ b/config-model/src/test/derived/rankproperties/rank-profiles.cfg @@ -1,6 +1,8 @@ rankprofile[].name "default" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" @@ -21,8 +23,10 @@ rankprofile[].fef.property[].value "0" rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" rankprofile[].fef.property[].value "true" rankprofile[].name "child" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" diff --git a/config-model/src/test/derived/rankproperties/rankproperties.sd b/config-model/src/test/derived/rankproperties/rankproperties.sd index 24977ae645f..db259f1daec 100644 --- a/config-model/src/test/derived/rankproperties/rankproperties.sd +++ b/config-model/src/test/derived/rankproperties/rankproperties.sd @@ -28,7 +28,8 @@ search rankproperties { expression: match } rank-properties { - $test:"foo" + $test1:"foo" + query(test2): 12.3 #$weight:1 } } diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd index e26358fef5a..339d147dfd0 100644 --- a/config-model/src/test/examples/rankpropvars.sd +++ b/config-model/src/test/examples/rankpropvars.sd @@ -55,13 +55,11 @@ document music { } field artist type string { - ## index-to: a indexing: index | summary } field year type int { indexing: attribute | summary - ## index-to: y } field url type uri {} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java index 86ccb816c10..0f29d2dda40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java @@ -10,8 +10,10 @@ import java.io.IOException; * @author bratseth */ public class RankPropertiesTestCase extends AbstractExportingTestCase { + @Test public void testRankProperties() throws IOException, ParseException { assertCorrectDeriving("rankproperties"); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index bcb197ed540..0b7dbf2eb21 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -435,7 +435,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } private VespaModel getIndexedSearchVespaModel(String xml) { - List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist"); + List<String> sds = generateSchemas("music", "album", "artist"); return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); } @@ -501,4 +501,55 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } } + public static String generateSchema(String name, String field1, String field2) { + return "schema " + name + " {" + + " document " + name + " {" + + " field " + field1 + " type string {\n" + + " indexing: index | summary\n" + + " summary: dynamic\n" + + " }\n" + + " field " + field2 + " type int {\n" + + " indexing: attribute | summary\n" + + " attribute: fast-access\n" + + " }\n" + + " field " + field2 + "_nfa type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " rank-profile staticrank inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }" + + " }" + + " rank-profile summaryfeatures inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " summary-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" + + " }" + + " rank-profile rankfeatures {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " rank-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " }" + + " }" + + "}"; + } + + public static List<String> generateSchemas(String ... sdNames) { + return generateSchemas(Arrays.asList(sdNames)); + } + + public static List<String> generateSchemas(List<String> sdNames) { + List<String> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2))); + i = i + 2; + } + return sds; + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 3cff04198bd..671a9f6660d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; @@ -16,11 +15,8 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; -import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; -import java.util.ArrayList; import java.util.List; import java.util.Arrays; import java.util.Map; @@ -35,92 +31,9 @@ public class DocumentDatabaseTestCase { private static final double SMALL = 0.00000000000001; - private static final String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" + - "<hosts> " + - " <host name='foo'>" + - " <alias>node0</alias>" + - " </host>" + - "</hosts>"; - - private String createVespaServices(List<String> sds, String mode) { - List<DocType> nameAndModes = new ArrayList<>(sds.size()); - for (String sd : sds) { - nameAndModes.add(DocType.create(sd, mode)); - } - return createVespaServicesXml(nameAndModes, ""); - } - private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) { - StringBuilder retval = new StringBuilder(); - retval.append("" + - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services version='1.0'>\n" + - "<admin version='2.0'>\n" + - " <adminserver hostalias='node0' />\n" + - "</admin>\n" + - "<container version='1.0'>\n" + - " <nodes>\n" + - " <node hostalias='node0'/>\n" + - " </nodes>\n" + - " <search/>\n" + - "</container>\n" + - "<content version='1.0' id='test'>\n" + - " <redundancy>1</redundancy>\n"); - retval.append(DocType.listToXml(nameAndModes)); - retval.append( - " <engine>\n" + - " <proton>\n" + - " <tuning>\n" + - " <searchnode>\n" + - xmlTuning + - " </searchnode>\n" + - " </tuning\n>" + - " </proton\n>" + - " </engine\n>" + - " <nodes>\n" + - " <node hostalias='node0' distribution-key='0'/>\n" + - " </nodes>\n" + - " </content>\n" + - "</services>\n"); - return retval.toString(); - } - - private ProtonConfig getProtonCfg(ContentSearchCluster cluster) { - ProtonConfig.Builder pb = new ProtonConfig.Builder(); - cluster.getConfig(pb); - return new ProtonConfig(pb); - } - - private void assertSingleSD(String mode) { - final List<String> sds = Arrays.asList("type1"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(); - IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); - ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); - String type1Id = "test/search/cluster.test/type1"; - ProtonConfig proton = getProtonCfg(contentSearchCluster); - assertEquals(1, proton.documentdb().size()); - assertEquals("type1", proton.documentdb(0).inputdoctypename()); - assertEquals(type1Id, proton.documentdb(0).configid()); - } - @Test public void requireThatWeCanHaveOneSDForIndexedMode() { - assertSingleSD("index"); - } - - private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { - return createModel(nameAndModes, xmlTuning, null); - } - - private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { - List<String> sds = new ArrayList<>(nameAndModes.size()); - for (DocType nameAndMode : nameAndModes) { - sds.add(nameAndMode.getType()); - } - var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), - ApplicationPackageUtils.generateSchemas(sds)); - return builder != null ? creator.create(builder) : creator.create(); + new SchemaTester().assertSingleSD("index"); } @Test @@ -178,9 +91,10 @@ public class DocumentDatabaseTestCase { if (featureFlagConcurrency != null) { properties.setFeedConcurrency(featureFlagConcurrency); } - VespaModel model = createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); + var tester = new SchemaTester(); + VespaModel model = tester.createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(global, proton.feeding().concurrency(), SMALL); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { @@ -190,11 +104,12 @@ public class DocumentDatabaseTestCase { @Test public void requireThatModeIsSet() { - VespaModel model = createModel(Arrays.asList(DocType.create("a", "index"), - DocType.create("b", "streaming"), - DocType.create("c", "store-only")), ""); + var tester = new SchemaTester(); + VespaModel model = tester.createModel(Arrays.asList(DocType.create("a", "index"), + DocType.create("b", "streaming"), + DocType.create("c", "store-only")), ""); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode()); assertEquals("a", proton.documentdb(0).inputdoctypename()); @@ -205,10 +120,11 @@ public class DocumentDatabaseTestCase { } private void verifyInitialDocumentCount(List<DocType> nameAndModes, String xmlTuning, List<Long> local) { + var tester = new SchemaTester(); assertEquals(nameAndModes.size(), local.size()); - VespaModel model = createModel(nameAndModes, xmlTuning); + VespaModel model = tester.createModel(nameAndModes, xmlTuning); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { assertEquals(local.get(i).longValue(), proton.documentdb(i).allocation().initialnumdocs()); @@ -240,14 +156,14 @@ public class DocumentDatabaseTestCase { assertEquals(attributeField, acfg.attribute(0).name()); assertEquals(attributeField+"_nfa", acfg.attribute(1).name()); RankProfilesConfig rcfg = model.getConfig(RankProfilesConfig.class, configId); - assertEquals(7, rcfg.rankprofile().size()); + assertEquals(6, rcfg.rankprofile().size()); } @Test public void testMultipleSchemas() { List<String> sds = List.of("type1", "type2", "type3"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"), - ApplicationPackageUtils.generateSchemas(sds)).create(); + var tester = new SchemaTester(); + var model = tester.createModel(sds); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); String type1Id = "test/search/cluster.test/type1"; @@ -255,7 +171,7 @@ public class DocumentDatabaseTestCase { String type3Id = "test/search/cluster.test/type3"; { assertEquals(3, indexedSearchCluster.getDocumentDbs().size()); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals("type1", proton.documentdb(0).inputdoctypename()); assertEquals(type1Id, proton.documentdb(0).configid()); @@ -295,9 +211,16 @@ public class DocumentDatabaseTestCase { @Test public void requireThatRelevantConfigIsAvailableForClusterSearcher() { - List<String> schemas = Arrays.asList("type1", "type2"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - ApplicationPackageUtils.generateSchemas(schemas)).create(); + String inputsProfile = + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " }" + + " }"; + List<String> schemas = List.of("type1", "type2"); + var tester = new SchemaTester(); + VespaModel model = tester.createModelWithRankProfile(inputsProfile, schemas); String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher"; { // documentdb-info config @@ -309,13 +232,13 @@ public class DocumentDatabaseTestCase { assertEquals("type1", db.name()); assertEquals(7, db.rankprofile().size()); - assertRankProfile(db, 0, "default", false, false); - assertRankProfile(db, 1, "unranked", false, false); - assertRankProfile(db, 2, "staticrank", false, false); - assertRankProfile(db, 3, "summaryfeatures", true, false); - assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); - assertRankProfile(db, 5, "rankfeatures", false, true); - var inputs = assertRankProfile(db, 6, "inputs", false, false); + tester.assertRankProfile(db, 0, "default", false, false); + tester.assertRankProfile(db, 1, "unranked", false, false); + tester.assertRankProfile(db, 2, "staticrank", false, false); + tester.assertRankProfile(db, 3, "summaryfeatures", true, false); + tester.assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); + tester.assertRankProfile(db, 5, "rankfeatures", false, true); + var inputs = tester.assertRankProfile(db, 6, "inputs", false, false); assertEquals(2, inputs.input().size()); assertEquals("query(foo)", inputs.input(0).name()); @@ -326,8 +249,8 @@ public class DocumentDatabaseTestCase { assertEquals(2, db.summaryclass().size()); assertEquals("default", db.summaryclass(0).name()); assertEquals("attributeprefetch", db.summaryclass(1).name()); - assertSummaryField(db, 0, 0, "f1", "longstring", true); - assertSummaryField(db, 0, 1, "f2", "integer", false); + tester.assertSummaryField(db, 0, 0, "f1", "longstring", true); + tester.assertSummaryField(db, 0, 1, "f2", "integer", false); } { // type2 DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(1); @@ -345,89 +268,10 @@ public class DocumentDatabaseTestCase { } } - /** Schema-info should contain all schemas, independent of clusters. */ - @Test - public void requireThatSchemaInfoIsAvailable() { - List<String> schemas = Arrays.asList("type1", "type2"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - ApplicationPackageUtils.generateSchemas(schemas)).create(); - assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model); - assertSchemaInfo("container", model); - } - - private void assertSchemaInfo(String configId, VespaModel model) { - { // schema-info config - SchemaInfoConfig dcfg = model.getConfig(SchemaInfoConfig.class, configId); - assertEquals(2, dcfg.schema().size()); - - { // type1 - SchemaInfoConfig.Schema schema = dcfg.schema(0); - assertEquals("type1", schema.name()); - - assertEquals(7, schema.rankprofile().size()); - assertRankProfile(schema, 0, "default", false, false); - assertRankProfile(schema, 1, "unranked", false, false); - assertRankProfile(schema, 2, "staticrank", false, false); - assertRankProfile(schema, 3, "summaryfeatures", true, false); - assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); - assertRankProfile(schema, 5, "rankfeatures", false, true); - var inputs = assertRankProfile(schema, 6, "inputs", false, false); - - assertEquals(2, inputs.input().size()); - assertEquals("query(foo)", inputs.input(0).name()); - assertEquals("tensor<float>(x[10])", inputs.input(0).type()); - assertEquals("query(bar)", inputs.input(1).name()); - assertEquals("tensor(key{},x[1000])", inputs.input(1).type()); - - // assertEquals(2, schema.summaryclass().size()); - // assertEquals("default", schema.summaryclass(0).name()); - // assertEquals("attributeprefetch", schema.summaryclass(1).name()); - // assertSummaryField(schema, 0, 0, "f1", "longstring", true); - // assertSummaryField(schema, 0, 1, "f2", "integer", false); - } - { // type2 - SchemaInfoConfig.Schema schema = dcfg.schema(1); - assertEquals("type2", schema.name()); - } - } - } - - private DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, - int index, - String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures) { - DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index); - assertEquals(name, rankProfile.name()); - assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); - assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); - return rankProfile; - } - - private SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, - int index, - String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures) { - SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index); - assertEquals(name, rankProfile.name()); - assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); - assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); - return rankProfile; - } - - private void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, - String name, String type, boolean dynamic) { - DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); - assertEquals(name, field.name()); - assertEquals(type, field.type()); - assertEquals(dynamic, field.dynamic()); - } - private void assertDocumentDBConfigAvailableForStreaming(String mode) { - final List<String> sds = Arrays.asList("type"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(); + List<String> sds = List.of("type"); + var tester = new SchemaTester(); + var model = tester.createModelWithMode(mode, sds); DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); assertEquals(1, dcfg.documentdb().size()); @@ -452,11 +296,11 @@ public class DocumentDatabaseTestCase { Map<String, List<String>> expectedAttributesMap, DeployState.Builder builder, long expectedMaxUnCommittedMemory) { - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(builder); + var tester = new SchemaTester(); + var model = tester.createModelWithMode(mode, sds, builder); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(sds.size(), proton.documentdb().size()); for (int i = 0; i < sds.size(); i++) { assertEquals(sds.get(i), proton.documentdb(i).inputdoctypename()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java new file mode 100644 index 00000000000..cf2135aea5a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class SchemaInfoTestCase { + + /** Schema-info should contain all schemas, independent of clusters. */ + @Test + public void requireThatSchemaInfoIsAvailable() { + List.of(1.0,2.0,3.0).toArray(new Double[3]); + String inputs = + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " query(myDouble1) double: 0.5" + + " query(myDouble2) tensor()" + + " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" + + " query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" + + " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" + + " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" + + " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" + + " }" + + " }"; + List<String> schemas = List.of("type1", "type2"); + var tester = new SchemaTester(); + var model = tester.createModelWithRankProfile(inputs, schemas); + assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model, tester); + assertSchemaInfo("container", model, tester); + } + + private void assertSchemaInfo(String configId, VespaModel model, SchemaTester tester) { + { + SchemaInfoConfig schemaInfoConfig = model.getConfig(SchemaInfoConfig.class, configId); + RankProfilesConfig rankProfilesConfig = model.getConfig(RankProfilesConfig.class, "test/search/cluster.test/type1"); + + assertEquals(2, schemaInfoConfig.schema().size()); + + { // type1 + SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(0); + assertEquals("type1", schema.name()); + + assertEquals(7, schema.rankprofile().size()); + tester.assertRankProfile(schema, 0, "default", false, false); + tester.assertRankProfile(schema, 1, "unranked", false, false); + tester.assertRankProfile(schema, 2, "staticrank", false, false); + tester.assertRankProfile(schema, 3, "summaryfeatures", true, false); + tester.assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); + tester.assertRankProfile(schema, 5, "rankfeatures", false, true); + + var schemaInfoProfile = tester.assertRankProfile(schema, 6, "inputs", false, false); + assertEquals(9, schemaInfoProfile.input().size()); + var rankProfilesProfile = rankProfilesConfig.rankprofile().get(6); + assertEquals("inputs", rankProfilesProfile.name()); + assertInput("query(foo)", "tensor<float>(x[10])", null, 0, schemaInfoProfile, rankProfilesProfile); + assertInput("query(bar)", "tensor(key{},x[1000])", null, 1, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble1)", "tensor()", "0.5", 2, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble2)", "tensor()", null, 3, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMap)", "tensor(key{})", "{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myVector)", "tensor(x[3])", "{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMatrix)", "tensor(x[2],y[3])", "{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 6, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed1)", "tensor(key{},x[2])", "{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 7, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile); + + assertEquals(2, schema.summaryclass().size()); + assertEquals("default", schema.summaryclass(0).name()); + assertEquals("attributeprefetch", schema.summaryclass(1).name()); + tester.assertSummaryField(schema, 0, 0, "f1", "longstring", true); + tester.assertSummaryField(schema, 0, 1, "f2", "integer", false); + } + { // type2 + SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(1); + assertEquals("type2", schema.name()); + } + } + } + + private void assertInput(String name, String type, String defaultValue, + int index, + SchemaInfoConfig.Schema.Rankprofile schemaInfoProfile, + RankProfilesConfig.Rankprofile rankProfilesProfile) { + assertEquals(name, schemaInfoProfile.input(index).name()); + assertEquals(type, schemaInfoProfile.input(index).type()); + if (defaultValue != null) { + boolean found = false; + for (var property : rankProfilesProfile.fef().property()) { + if (property.name().equals(name)) { + assertEquals(defaultValue, property.value()); + found = true; + } + } + if ( ! found) + fail("Missing property " + name); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java new file mode 100644 index 00000000000..7a077c1a0aa --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java @@ -0,0 +1,220 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.content.utils.DocType; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SchemaTester { + + private static final String vespaHosts = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<hosts> " + + " <host name='foo'>" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + private String createVespaServices(List<String> sds, String mode) { + List<DocType> nameAndModes = new ArrayList<>(sds.size()); + for (String sd : sds) { + nameAndModes.add(DocType.create(sd, mode)); + } + return createVespaServicesXml(nameAndModes, ""); + } + private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) { + StringBuilder retval = new StringBuilder(); + retval.append("" + + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + "<admin version='2.0'>\n" + + " <adminserver hostalias='node0' />\n" + + "</admin>\n" + + "<container version='1.0'>\n" + + " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " <search/>\n" + + "</container>\n" + + "<content version='1.0' id='test'>\n" + + " <redundancy>1</redundancy>\n"); + retval.append(DocType.listToXml(nameAndModes)); + retval.append( + " <engine>\n" + + " <proton>\n" + + " <tuning>\n" + + " <searchnode>\n" + + xmlTuning + + " </searchnode>\n" + + " </tuning\n>" + + " </proton\n>" + + " </engine\n>" + + " <nodes>\n" + + " <node hostalias='node0' distribution-key='0'/>\n" + + " </nodes>\n" + + " </content>\n" + + "</services>\n"); + return retval.toString(); + } + + ProtonConfig getProtonCfg(ContentSearchCluster cluster) { + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + cluster.getConfig(pb); + return new ProtonConfig(pb); + } + + void assertSingleSD(String mode) { + List<String> sds = List.of("type1"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), + generateSchemas("", sds)).create(); + IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); + String type1Id = "test/search/cluster.test/type1"; + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(1, proton.documentdb().size()); + assertEquals("type1", proton.documentdb(0).inputdoctypename()); + assertEquals(type1Id, proton.documentdb(0).configid()); + } + + VespaModel createModel(List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas("", schemas)).create(); + } + + VespaModel createModelWithRankProfile(String rankProfile, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas(rankProfile, schemas)).create(); + } + + VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { + return createModel(nameAndModes, xmlTuning, null); + } + + VespaModel createModelWithMode(String mode, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, mode), + generateSchemas("", schemas)).create(); + + } + + VespaModel createModelWithMode(String mode, List<String> schemas, DeployState.Builder builder) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, mode), + generateSchemas("", schemas)).create(builder); + } + + VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { + List<String> sds = new ArrayList<>(nameAndModes.size()); + for (DocType nameAndMode : nameAndModes) { + sds.add(nameAndMode.getType()); + } + var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), + generateSchemas("", sds)); + return builder != null ? creator.create(builder) : creator.create(); + } + + public static String generateSchema(String name, String field1, String field2, String rankProfile) { + return "schema " + name + " {" + + " document " + name + " {" + + " field " + field1 + " type string {\n" + + " indexing: index | summary\n" + + " summary: dynamic\n" + + " }\n" + + " field " + field2 + " type int {\n" + + " indexing: attribute | summary\n" + + " attribute: fast-access\n" + + " }\n" + + " field " + field2 + "_nfa type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " rank-profile staticrank inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }" + + " }" + + " rank-profile summaryfeatures inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " summary-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" + + " }" + + " rank-profile rankfeatures {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " rank-features: attribute(" + field2 + ")" + + " }" + + rankProfile + + "}"; + } + + public static List<String> generateSchemas(String rankProfile, String ... sdNames) { + return generateSchemas(rankProfile, Arrays.asList(sdNames)); + } + + public static List<String> generateSchemas(String rankProfile, List<String> sdNames) { + List<String> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), rankProfile)); + i = i + 2; + } + return sds; + } + + DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, + int index, + String name, + boolean hasSummaryFeatures, + boolean hasRankFeatures) { + DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index); + assertEquals(name, rankProfile.name()); + assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); + return rankProfile; + } + + SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, + int index, + String name, + boolean hasSummaryFeatures, + boolean hasRankFeatures) { + SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index); + assertEquals(name, rankProfile.name()); + assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); + return rankProfile; + } + + void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, + String name, String type, boolean dynamic) { + DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); + assertEquals(name, field.name()); + assertEquals(type, field.type()); + assertEquals(dynamic, field.dynamic()); + } + + void assertSummaryField(SchemaInfoConfig.Schema schema, int summaryClassIndex, int fieldIndex, + String name, String type, boolean dynamic) { + SchemaInfoConfig.Schema.Summaryclass.Fields field = schema.summaryclass(summaryClassIndex).fields(fieldIndex); + assertEquals(name, field.name()); + assertEquals(type, field.type()); + assertEquals(dynamic, field.dynamic()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index 952de2a6f40..df611deb72a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -7,6 +7,7 @@ import java.util.List; /** * For testing purposes only. + * * @author geirst */ public class ApplicationPackageUtils { diff --git a/config/src/vespa/config/file_acquirer/file_acquirer.cpp b/config/src/vespa/config/file_acquirer/file_acquirer.cpp index 9cf280566e0..048e9544dab 100644 --- a/config/src/vespa/config/file_acquirer/file_acquirer.cpp +++ b/config/src/vespa/config/file_acquirer/file_acquirer.cpp @@ -5,7 +5,6 @@ #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/transport.h> -#include <vespa/vespalib/util/size_literals.h> #include <vespa/fastos/thread.h> #include <vespa/log/log.h> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index fbad212b85f..1042c38f995 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -207,6 +207,7 @@ public class ModelContextImpl implements ModelContext { private final boolean experimentalSdParsing; private final boolean enableBitVectors; private final Architecture adminClusterArchitecture; + private final boolean enableProxyProtocolMixedMode; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -251,6 +252,7 @@ public class ModelContextImpl implements ModelContext { this.experimentalSdParsing = flagValue(source, appId, version, Flags.EXPERIMENTAL_SD_PARSING); this.enableBitVectors = flagValue(source, appId, version, Flags.ENABLE_BIT_VECTORS); this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); + this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -297,6 +299,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean experimentalSdParsing() { return experimentalSdParsing; } @Override public boolean enableBitVectors() { return this.enableBitVectors; } @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } + @Override public boolean enableProxyProtocolMixedMode() { return enableProxyProtocolMixedMode; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 00c3a1d1aae..95494190734 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -47,7 +47,7 @@ public class Coverage { protected Coverage(long docs, int nodes, boolean full, int resultSets) { this(docs, docs, nodes, resultSets, full ? FullCoverageDefinition.EXPLICITLY_FULL - : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); + : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); } private Coverage(long docs, long active, int nodes, int resultSets, FullCoverageDefinition fullReason) { @@ -168,9 +168,8 @@ public class Coverage { } /** - * An int between 0 (inclusive) and 100 (inclusive) representing how many - * percent coverage the result sets this Coverage instance contains information - * about had. + * An int between 0 (inclusive) and 100 (inclusive) representing the + * percent coverage of the result sets this instance contains information about. */ public int getResultPercentage() { if (getResultSets() == 0) { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index ab764e97fa0..2c25f38437a 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -42,7 +42,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import com.yahoo.log.impl.InitializeLog; +import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigKey; @@ -109,7 +109,7 @@ public final class ConfiguredApplication implements Application { private volatile boolean shutdownReconfiguration = false; static { - InitializeLog.init(); + LogSetup.initVespaLogging("Container"); log.log(Level.INFO, "Starting jdisc" + (Vtag.currentVersion.isEmpty() ? "" : " at version " + Vtag.currentVersion)); installBouncyCastleSecurityProvider(); } @@ -460,6 +460,7 @@ public final class ConfiguredApplication implements Application { slobrokConfigSubscriber.ifPresent(SlobrokConfigSubscriber::shutdown); Container.get().shutdown(); unregisterInSlobrok(); + LogSetup.cleanup(); shutdownDeadline.cancel(); log.info("Destroy: Finished"); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 19e0e441359..abd23c1822d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -147,7 +147,7 @@ public class QueryProfileProperties extends Properties { private String toShortString(Object value) { if ( ! (value instanceof Tensor)) return value.toString(); - return ((Tensor)value).toShortString(); + return ((Tensor)value).toAbbreviatedString(); } private Object convertByType(CompoundName name, Object value, Map<String, String> context) { diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 5074a520a4e..aa561f58d7a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -26,7 +26,7 @@ public class Coverage extends com.yahoo.container.handler.Coverage { /** * Will set number of documents present in ideal state * - * @param soonActive Number of documents active in ideal state + * @param soonActive number of documents active in ideal state * @return self for chaining */ @Beta @@ -35,10 +35,11 @@ public class Coverage extends com.yahoo.container.handler.Coverage { /** * Will set the reasons for degraded coverage as reported by vespa backend. * - * @param degradedReason Reason for degradation + * @param degradedReason reason for degradation * @return self for chaining */ public Coverage setDegradedReason(int degradedReason) { this.degradedReason = degradedReason; return this; } public Coverage setNodesTried(int nodesTried) { super.setNodesTried(nodesTried); return this; } + } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index bff0e881c3d..12f71c4fc84 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -396,6 +396,13 @@ public class Flags { "Takes effect immediately", APPLICATION_ID); + public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( + "enable-proxy-protocol-mixed-mode", true, + List.of("tokle"), "2022-05-09", "2022-07-01", + "Enable or disable proxy protocol mixed mode", + "Takes effect on redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java index 601a96da24d..952e8dd5f00 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java @@ -16,8 +16,6 @@ import com.yahoo.jdisc.application.OsgiHeader; import com.yahoo.jdisc.service.ContainerNotReadyException; import com.yahoo.jdisc.service.CurrentContainer; import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; -import com.yahoo.log.impl.InitializeLog; -import com.yahoo.log.LogSetup; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleException; @@ -48,7 +46,6 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C private ApplicationInUseTracker applicationInUseTracker; public ApplicationLoader(OsgiFramework osgiFramework, Iterable<? extends Module> guiceModules) { - InitializeLog.init(); this.osgiFramework = osgiFramework; this.guiceModules.install(new ApplicationEnvironmentModule(this)); this.guiceModules.installAll(guiceModules); @@ -201,7 +198,6 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C try { watchdog.close(); osgiFramework.stop(); - LogSetup.cleanup(); } catch (BundleException | InterruptedException e) { e.printStackTrace(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index dff72fe81f1..a5edd3bd74d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -110,6 +110,7 @@ public class NodeAdminImpl implements NodeAdmin { if (!isSuspended) { Runtime runtime = Runtime.getRuntime(); + runtime.gc(); long freeMemory = runtime.freeMemory(); long totalMemory = runtime.totalMemory(); long usedMemory = totalMemory - freeMemory; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp index a3a180a74df..3aecbc3ca0d 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp @@ -2,6 +2,7 @@ #include "lidvectorcontext.h" #include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/vespalib/objects/nbostream.h> #include <cassert> @@ -9,6 +10,7 @@ LOG_SETUP(".proton.feedoperation.lidvectorcontext"); using search::BitVector; +using search::AllocatedBitVector; namespace proton { @@ -73,7 +75,7 @@ LidVectorContext::deserialize(vespalib::nbostream &is) LOG(debug, "deserialize: format = %d", format); // Use of bitvector when > 1/32 of docs if (format == BITVECTOR) { - BitVector::UP bitVector = BitVector::create(_docIdLimit); + auto bitVector = std::make_unique<AllocatedBitVector>(_docIdLimit); is >> *bitVector; uint32_t sz(bitVector->size()); assert(sz == _docIdLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 4e272e1af4c..8cc64f1b0de 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -319,7 +319,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot MaintenanceConfigSP newMaintenanceConfig; std::shared_ptr<const ThreadingServiceConfig> old_threading_service_config; std::shared_ptr<const AllocConfig> old_alloc_config; - constexpr vespalib::duration file_resolve_timeout = 60s * 10; + constexpr vespalib::duration file_resolve_timeout = 60min; if (!_ignoreForwardedConfig) { if (!(_bootstrapConfig->getDocumenttypesConfigSP() && @@ -352,6 +352,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) { newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId); } + vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) { RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP( snapshot.getConfig<RankingConstantsConfig>(_configId)); @@ -359,7 +360,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot RankingConstants::Vector constants; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const RankingConstantsConfig::Constant &rc : newRankingConstantsConfig->constant) { auto desc = fmt("name='%s', type='%s'", rc.name.c_str(), rc.type.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); @@ -375,7 +375,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot RankingExpressions expressions; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const RankingExpressionsConfig::Expression &rc : newRankingExpressionsConfig->expression) { auto desc = fmt("name='%s'", rc.name.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); @@ -391,7 +390,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot OnnxModels::Vector models; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const OnnxModelsConfig::Model &rc : newOnnxModelsConfig->model) { auto desc = fmt("name='%s'", rc.name.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index e811e68f4b5..0658287f29b 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/vespalib/stllike/cache_stats.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> @@ -95,7 +95,7 @@ public: uint32_t _visitCount; uint32_t _visitRmCount; uint32_t _docIdLimit; - BitVector::UP _valid; + std::unique_ptr<AllocatedBitVector> _valid; bool _before; MyVisitorBase(DocumentTypeRepo &repo, uint32_t docIdLimit, bool before); @@ -106,7 +106,7 @@ MyVisitorBase::MyVisitorBase(DocumentTypeRepo &repo, uint32_t docIdLimit, bool b _visitCount(0u), _visitRmCount(0u), _docIdLimit(docIdLimit), - _valid(BitVector::create(docIdLimit)), + _valid(std::make_unique<AllocatedBitVector>(docIdLimit)), _before(before) { } @@ -216,7 +216,7 @@ struct Fixture std::unique_ptr<LogDocumentStore> _store; uint64_t _syncToken; uint32_t _docIdLimit; - BitVector::UP _valid; + std::unique_ptr<AllocatedBitVector> _valid; Fixture(); ~Fixture(); @@ -246,7 +246,7 @@ Fixture::Fixture() _store(), _syncToken(0u), _docIdLimit(0u), - _valid(BitVector::create(0u)) + _valid(std::make_unique<AllocatedBitVector>(0u)) { rmdir(); mkdir(); diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index 8d137530dd0..d6aa136e15f 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -122,7 +122,7 @@ void FlagAttributeT<B>::setNewValues(DocId doc, const std::vector<typename B::WT BitVector * bv = _bitVectors[offset]; if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); - _bitVectorStore[offset] = BitVector::create(_bitVectorSize); + _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; ensureGuardBit(*bv); @@ -139,7 +139,7 @@ FlagAttributeT<B>::setNewBVValue(DocId doc, multivalue::ValueType_t<typename B:: BitVector * bv = _bitVectors[offset]; if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); - _bitVectorStore[offset] = BitVector::create(_bitVectorSize); + _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; ensureGuardBit(*bv); @@ -210,11 +210,11 @@ FlagAttributeT<B>::resizeBitVectors(uint32_t neededSize) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = neededSize + (neededSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); - for (BitVector * bv : _bitVectors) { + for (size_t i(0), m(_bitVectors.size()); i < m; i++) { + BitVector *bv = _bitVectors[i]; if (bv != nullptr) { - vespalib::GenerationHeldBase::UP hold(bv->grow(newSize)); + _bitVectorStore[i]->extend(newSize); ensureGuardBit(*bv); - _bitVectorHolder.hold(std::move(hold)); } } _bitVectorSize = newSize; diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index f88f46bf974..c4f4980e372 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -3,6 +3,7 @@ #include "multinumericattribute.h" #include "multi_numeric_search_context.h" +#include <vespa/searchlib/common/growablebitvector.h> namespace search { @@ -34,10 +35,10 @@ private: void removeOldGenerations(vespalib::GenerationHandler::generation_t firstUsed) override; uint32_t getOffset(int8_t value) const { return value + 128; } - vespalib::GenerationHolder _bitVectorHolder; - std::vector<std::shared_ptr<BitVector> > _bitVectorStore; - std::vector<BitVector *> _bitVectors; - uint32_t _bitVectorSize; + vespalib::GenerationHolder _bitVectorHolder; + std::vector<std::shared_ptr<GrowableBitVector> > _bitVectorStore; + std::vector<BitVector *> _bitVectors; + uint32_t _bitVectorSize; }; typedef FlagAttributeT<FlagBaseImpl> FlagAttribute; diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index ab56317db6f..f37968c70a0 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -6,10 +6,6 @@ namespace search { -using vespalib::GenerationHeldBase; -using vespalib::GenerationHeldAlloc; -using vespalib::GenerationHolder; - namespace { size_t computeCapacity(size_t capacity, size_t allocatedBytes) { @@ -131,30 +127,4 @@ AllocatedBitVector::operator=(const BitVector & rhs) return *this; } -GenerationHeldBase::UP -AllocatedBitVector::grow(Index newSize, Index newCapacity) -{ - assert(newCapacity >= newSize); - GenerationHeldBase::UP ret; - if (newCapacity != capacity()) { - AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size(), &_alloc); - if (newSize > size()) { - tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. - } - ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc); - swap(tbv); - } else { - if (newSize > size()) { - Range clearRange(size(), newSize); - setSize(newSize); - clearIntervalNoInvalidation(clearRange); - } else { - clearIntervalNoInvalidation(Range(newSize, size())); - setSize(newSize); - updateCount(); - } - } - return ret; -} - } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index 2223c7b4701..5106717dad0 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -6,6 +6,7 @@ namespace search { +class GrowableBitVector; class BitVectorTest; /** @@ -39,7 +40,7 @@ public: AllocatedBitVector(const BitVector &other); AllocatedBitVector(const AllocatedBitVector &other); - virtual ~AllocatedBitVector(); + ~AllocatedBitVector() override; AllocatedBitVector &operator=(const AllocatedBitVector &other); AllocatedBitVector &operator=(const BitVector &other); @@ -57,9 +58,7 @@ public: * * @param newLength the new length of the bit vector (in bits) */ - void resize(Index newLength) override; - - GenerationHeldBase::UP grow(Index newLength, Index newCapacity) override; + void resize(Index newLength); protected: Index _capacityBits; @@ -67,6 +66,7 @@ protected: private: friend class BitVectorTest; + friend class GrowableBitVector; void swap(AllocatedBitVector & rhs) { std::swap(_capacityBits, rhs._capacityBits); _alloc.swap(rhs._alloc); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index d5b21a38c3f..36ebc6597fa 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -2,7 +2,6 @@ #include "bitvector.h" #include "allocatedbitvector.h" -#include "growablebitvector.h" #include "partialbitvector.h" #include <vespa/searchlib/util/file_settings.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> @@ -299,18 +298,6 @@ BitVector::hasTrueBitsInternal() const } ////////////////////////////////////////////////////////////////////// -// Set new length. Destruction of content -////////////////////////////////////////////////////////////////////// -void -BitVector::resize(Index) -{ - LOG_ABORT("should not be reached"); -} -GenerationHeldBase::UP -BitVector::grow(Index, Index ) -{ - LOG_ABORT("should not be reached"); -} size_t BitVector::getFileBytes(Index bits) @@ -383,12 +370,6 @@ BitVector::create(const BitVector & rhs) return std::make_unique<AllocatedBitVector>(rhs); } -BitVector::UP -BitVector::create(Index numberOfElements, Index newCapacity, GenerationHolder &generationHolder) -{ - return std::make_unique<GrowableBitVector>(numberOfElements, newCapacity, generationHolder); -} - MMappedBitVector::MMappedBitVector(Index numberOfElements, FastOS_FileInterface &file, int64_t offset, Index doccount) : BitVector() @@ -425,7 +406,7 @@ operator<<(nbostream &out, const BitVector &bv) nbostream & -operator>>(nbostream &in, BitVector &bv) +operator>>(nbostream &in, AllocatedBitVector &bv) { uint64_t size; uint64_t cachedHits; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 53e9ed45d5e..7fea95661d7 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -18,6 +18,7 @@ class FastOS_FileInterface; namespace search { class PartialBitVector; +class AllocatedBitVector; class BitVector : protected BitWord { @@ -253,11 +254,6 @@ public: return getFileBytes(size()); } - virtual void resize(Index newLength); - - virtual GenerationHeldBase::UP grow(Index newLength, Index newCapacity); - GenerationHeldBase::UP grow(Index newLength) { return grow(newLength, newLength); } - /** * This will create the appropriate vector. * @@ -271,7 +267,6 @@ public: static UP create(const BitVector & org, Index start, Index end); static UP create(Index numberOfElements); static UP create(const BitVector & rhs); - static UP create(Index newSize, Index newCapacity, GenerationHolder &generationHolder); protected: using Alloc = vespalib::alloc::Alloc; VESPA_DLL_LOCAL BitVector(void * buf, Index start, Index end); @@ -384,14 +379,14 @@ protected: friend vespalib::nbostream & operator<<(vespalib::nbostream &out, const BitVector &bv); friend vespalib::nbostream & - operator>>(vespalib::nbostream &in, BitVector &bv); + operator>>(vespalib::nbostream &in, AllocatedBitVector &bv); }; vespalib::nbostream & operator<<(vespalib::nbostream &out, const BitVector &bv); vespalib::nbostream & -operator>>(vespalib::nbostream &in, BitVector &bv); +operator>>(vespalib::nbostream &in, AllocatedBitVector &bv); template <typename T> void BitVector::andNotWithT(T it) { diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index 7c29945292e..7a548b3e7ac 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -7,8 +7,35 @@ namespace search { using vespalib::GenerationHeldBase; +using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; +GenerationHeldBase::UP +GrowableBitVector::grow(Index newSize, Index newCapacity) +{ + assert(newCapacity >= newSize); + GenerationHeldBase::UP ret; + if (newCapacity != capacity()) { + AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size(), &_alloc); + if (newSize > size()) { + tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. + } + ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc); + swap(tbv); + } else { + if (newSize > size()) { + Range clearRange(size(), newSize); + setSize(newSize); + clearIntervalNoInvalidation(clearRange); + } else { + clearIntervalNoInvalidation(Range(newSize, size())); + setSize(newSize); + updateCount(); + } + } + return ret; +} + GrowableBitVector::GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder, const Alloc* init_alloc) diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index 71261e130b6..8773f1573d6 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -16,6 +16,8 @@ public: bool shrink(Index newCapacity); bool extend(Index newCapacity); private: + GenerationHeldBase::UP grow(Index newLength, Index newCapacity); + VESPA_DLL_LOCAL bool hold(GenerationHeldBase::UP v); GenerationHolder &_generationHolder; }; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 6044666ebf8..f7be61946ba 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -787,7 +787,10 @@ "public", "abstract" ], - "methods": [], + "methods": [ + "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(float[])", + "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(double[])" + ], "fields": [] }, "com.yahoo.tensor.IndexedTensor$Builder": { @@ -903,7 +906,8 @@ "public java.util.Map cells()", "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public bridge synthetic com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)" ], @@ -954,7 +958,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)" ], "fields": [] @@ -1046,7 +1051,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public long denseSubspaceSize()", "public static com.yahoo.tensor.TensorType createPartialType(com.yahoo.tensor.TensorType$Value, java.util.List)" @@ -1092,6 +1098,7 @@ ], "methods": [ "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, java.lang.String)", + "public com.yahoo.tensor.TensorType type()", "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, long)", "public com.yahoo.tensor.Tensor$Builder value(double)", "public com.yahoo.tensor.Tensor$Builder value(float)" @@ -1233,9 +1240,11 @@ "public java.util.List largest()", "public java.util.List smallest()", "public abstract java.lang.String toString()", - "public abstract java.lang.String toShortString()", - "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, long)", - "public static java.lang.String contentToString(com.yahoo.tensor.Tensor, long)", + "public abstract java.lang.String toString(boolean, boolean)", + "public abstract java.lang.String toAbbreviatedString()", + "public java.lang.String toShortString()", + "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, boolean, boolean, long)", + "public static java.lang.String valueToString(com.yahoo.tensor.Tensor, boolean, long)", "public abstract boolean equals(java.lang.Object)", "public abstract int hashCode()", "public static boolean equals(com.yahoo.tensor.Tensor, com.yahoo.tensor.Tensor)", @@ -1256,8 +1265,10 @@ ], "methods": [ "public void <init>(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String)", "public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String, java.lang.String)", "public com.yahoo.tensor.TensorAddress$Builder copy()", + "public com.yahoo.tensor.TensorType type()", "public com.yahoo.tensor.TensorAddress build()" ], "fields": [] diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 89eefeced56..c4316eb334a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -219,21 +219,26 @@ public abstract class IndexedTensor implements Tensor { } @Override - public String toString() { return toString(Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } - Indexes indexes = Indexes.of(dimensionSizes); + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms || type.rank() == 0 || type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - StringBuilder b = new StringBuilder(type.toString()).append(":"); + Indexes indexes = Indexes.of(dimensionSizes); + StringBuilder b = new StringBuilder(); + if (withType) + b.append(type).append(":"); indexedBlockToString(this, indexes, maxCells, b); return b.toString(); } @@ -438,7 +443,7 @@ public abstract class IndexedTensor implements Tensor { this.sizes = sizes; } - BoundBuilder fill(float[] values) { + public BoundBuilder fill(float[] values) { long index = 0; for (float value : values) { cellByDirectIndex(index++, value); @@ -446,7 +451,7 @@ public abstract class IndexedTensor implements Tensor { return this; } - BoundBuilder fill(double[] values) { + public BoundBuilder fill(double[] values) { long index = 0; for (double value : values) { cellByDirectIndex(index++, value); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index ad945ed18bf..946d8fe0f4a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -72,11 +72,18 @@ public class MappedTensor implements Tensor { public int hashCode() { return cells.hashCode(); } @Override - public String toString() { return Tensor.toStandardString(this, Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return Tensor.toStandardString(this, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { return toString(withType, shortForms, Long.MAX_VALUE); } + + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + return Tensor.toStandardString(this, withType, shortForms, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 56bd94a86e9..d2fed9b96f9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -145,23 +145,27 @@ public class MixedTensor implements Tensor { @Override public String toString() { - return toString(Long.MAX_VALUE); + return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) - return Tensor.toStandardString(this, maxCells); - if (type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms + || type.rank() == 0 + || type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty()) + || type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - return type + ":" + index.contentToString(this, maxCells); + return (withType ? type + ":" : "") + index.contentToString(this, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 06e7b010a7a..8a84e97fe05 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -32,7 +32,6 @@ import java.util.Set; import java.util.function.DoubleBinaryOperator; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; -import java.util.stream.Collectors; import static com.yahoo.tensor.functions.ScalarFunctions.Hamming; @@ -316,8 +315,22 @@ public interface Tensor { @Override String toString(); + /** + * Returns this tensor on the + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * + * @param withType whether to prefix the value by the type of this + * @param shortForms whether to use short forms where applicable, or always using the verbose form + */ + String toString(boolean withType, boolean shortForms); + /** Returns an abbreviated string representation of this tensor suitable for human-readable messages */ - String toShortString(); + String toAbbreviatedString(); + + // TODO: Remove on Vespa 8 + /** @deprecated use toAbbreviatedString */ + @Deprecated + default String toShortString() { return toAbbreviatedString(); } /** * Call this from toString in implementations to return this tensor on the @@ -325,15 +338,16 @@ public interface Tensor { * (toString cannot be a default method because default methods cannot override super methods). * * @param tensor the tensor to return the standard string format of + * @param withType whether the type should be prepended to the content * @param maxCells the max number of cells to output, after which just , "..." is output to represent the rest * of the cells * @return the tensor on the standard string format */ - static String toStandardString(Tensor tensor, long maxCells) { - return tensor.type() + ":" + contentToString(tensor, maxCells); + static String toStandardString(Tensor tensor, boolean withType, boolean shortForms, long maxCells) { + return (withType ? tensor.type() + ":" : "") + valueToString(tensor, shortForms, maxCells); } - static String contentToString(Tensor tensor, long maxCells) { + static String valueToString(Tensor tensor, boolean shortForms, long maxCells) { var cellEntries = new ArrayList<>(tensor.cells().entrySet()); cellEntries.sort(Map.Entry.comparingByKey()); if (tensor.type().dimensions().isEmpty()) { @@ -345,7 +359,7 @@ public interface Tensor { for (; i < cellEntries.size() && i < maxCells; i++) { if (i > 0) b.append(", "); - b.append(cellToString(cellEntries.get(i), tensor.type())); + b.append(cellToString(cellEntries.get(i), tensor.type(), shortForms)); } if (i == maxCells && i < tensor.size()) b.append(", ..."); @@ -353,8 +367,9 @@ public interface Tensor { return b.toString(); } - private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type) { - return (type.rank() > 1 ? cell.getKey().toString(type) : TensorAddress.labelToString(cell.getKey().label(0))) + + private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type, boolean shortForms) { + return (shortForms && type.rank() == 1 ? TensorAddress.labelToString(cell.getKey().label(0)) + : cell.getKey().toString(type) ) + ":" + cell.getValue(); } @@ -533,7 +548,7 @@ public interface Tensor { return IndexedTensor.Builder.of(type, dimensionSizes); } - /** Returns the type this is building */ + /** Returns the type of the tensor this is building */ TensorType type(); /** Return a cell builder */ @@ -578,6 +593,9 @@ public interface Tensor { return this; } + /** Returns the type of the tensor this cell is build for. */ + public TensorType type() { return tensorBuilder.type(); } + public CellBuilder label(String dimension, long label) { return label(dimension, String.valueOf(label)); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 27e752f1180..d9ab67d6c5f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -168,7 +168,7 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } - /** Supports building of a tensor address */ + /** Builder of a tensor address */ public static class Builder { private final TensorType type; @@ -184,6 +184,18 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } /** + * Adds the label to the only dimension of this. + * + * @throws IllegalArgumentException if this does not have exactly one dimension + */ + public Builder add(String label) { + if (type.rank() != 1) + throw new IllegalArgumentException("Cannot add a label without explicit dimension to a tensor of type " + type); + add(type.dimensions().get(0).name(), label); + return this; + } + + /** * Adds a label in a dimension to this. * * @return this for convenience @@ -203,6 +215,9 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { return new Builder(type, Arrays.copyOf(labels, labels.length)); } + /** Returns the type of the tensor this address is being built for. */ + public TensorType type() { return type; } + public TensorAddress build() { for (int i = 0; i < labels.length; i++) if (labels[i] == null) diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 2067d7a8492..920f8512c53 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -65,24 +65,24 @@ public class TensorTestCase { @Test public void testToShortString() { assertEquals("tensor(x[10]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", - Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toShortString()); + Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toAbbreviatedString()); assertEquals("tensor(x[14]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, ...]", - Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toShortString()); + Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toAbbreviatedString()); assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0, {d1:l1,d2:l3}:6.0, ...}", Tensor.from("{{d1:l1,d2:l1}:6, {d2:l2,d1:l1}:6, {d2:l3,d1:l1}:6, {d2:l4,d1:l1}:6, {d2:l5,d1:l1}:6," + " {d2:l6,d1:l1}:6, {d2:l7,d1:l1}:6, {d2:l8,d1:l1}:6, {d2:l9,d1:l1}:6, {d2:l2,d1:l2}:6," + - " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toShortString()); + " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},n{},x[3]):{{m:k1,n:k1,x:0}:0.0, {m:k1,n:k1,x:1}:1.0, {m:k1,n:k1,x:2}:2.0, ...}", Tensor.from("tensor(m{},n{},x[3]):" + "{{m:k1,n:k1,x:0}:0, {m:k1,n:k1,x:1}:1, {m:k1,n:k1,x:2}:2, " + " {m:k2,n:k1,x:0}:0, {m:k2,n:k1,x:1}:1, {m:k2,n:k1,x:2}:2, " + - " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toShortString()); + " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toAbbreviatedString()); assertEquals("tensor(m{},x[2],y[2]):{k1:[[0.0, 1.0], [2.0, 3.0]], k2:[[0.0, ...}", - Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toShortString()); + Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toAbbreviatedString()); } @Test diff --git a/vespalog/src/main/java/com/yahoo/log/LogSetup.java b/vespalog/src/main/java/com/yahoo/log/LogSetup.java index d285382849e..67fb7f71a2d 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogSetup.java +++ b/vespalog/src/main/java/com/yahoo/log/LogSetup.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import com.yahoo.log.impl.LogUtils; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -23,7 +25,7 @@ import java.util.logging.Logger; @SuppressWarnings("removal") public class LogSetup { - private static final Timer taskRunner = new Timer(true); + private static Timer taskRunner = new Timer(true); /** * A global task thread diff --git a/vespalog/src/main/java/com/yahoo/log/impl/InitializeLog.java b/vespalog/src/main/java/com/yahoo/log/impl/InitializeLog.java deleted file mode 100644 index 9b309916469..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/impl/InitializeLog.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.yahoo.log.impl; - -import com.yahoo.log.LogSetup; - -/** - * Sets up Vespa logging. Call a setup method to set up this. - * - * @author baldersheim - */ -public class InitializeLog { - static { - LogSetup.initVespaLogging("unused-default"); - } - - /** - * Do not delete this method even if it's empty. - * Calling this methods forces this class to be loaded, - * which runs the static block. - */ - public static void init() { } -} diff --git a/vespalog/src/main/java/com/yahoo/log/impl/LogUtils.java b/vespalog/src/main/java/com/yahoo/log/impl/LogUtils.java index e8d7b17abff..8c182625c85 100644 --- a/vespalog/src/main/java/com/yahoo/log/impl/LogUtils.java +++ b/vespalog/src/main/java/com/yahoo/log/impl/LogUtils.java @@ -8,6 +8,7 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; * @author Bjorn Borud * @author arnej27959 * @author bjorncs + * TODO remove "public" keyword, should be package private */ public class LogUtils { public static boolean empty(String s) { diff --git a/vespalog/src/main/java/com/yahoo/log/impl/package-info.java b/vespalog/src/main/java/com/yahoo/log/impl/package-info.java deleted file mode 100644 index ac6fe22b9cf..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/impl/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.log.impl; - -import com.yahoo.osgi.annotation.ExportPackage; |