diff options
250 files changed, 1756 insertions, 971 deletions
diff --git a/.copr/Makefile b/.copr/Makefile index d515053b8bc..bb37e1c4401 100644 --- a/.copr/Makefile +++ b/.copr/Makefile @@ -8,6 +8,7 @@ SPECFILE := $(SPECDIR)/vespa-$(VESPA_VERSION).spec deps: dnf install -y git rpmdevtools + git config --global --add safe.directory $$(realpath $(TOP)/..) srpm: VESPA_VERSION = $$(git tag --points-at HEAD | grep -oP "\d+\.\d+\.\d+" | sort -V | tail -1) srpm: deps diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java index 0661f73b497..f3568caac04 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CertificateExpiryMetricUpdater.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java index 322a262206a..c659c454420 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.security.KeyUtils; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 962a2c27dab..147a05bc27e 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java index ba357e44e09..c33974abf7a 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.net.HostName; import com.yahoo.vespa.athenz.api.AthenzService; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java index 941d8d46dcd..c1dd70d7656 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java index 148d52c3f33..fc21f8271c4 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.common.net.InetAddresses; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 85fdd6e6d43..9f4b1a9d01c 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.ca.restapi; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java index 5f606c68c1e..f0827aa79d1 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.apps.clustercontroller; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscMetricWrapper; diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index a0e290de172..bf38cf52aea 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.apps.clustercontroller; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vdslib.distribution.Distribution; diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java index 517f1f9a02c..5ef4c544bc8 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.apps.clustercontroller; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscHttpRequestHandler; diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java index 368d9e52b52..24d4a67fcac 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StatusHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.apps.clustercontroller; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscHttpRequestHandler; public class StatusHandler extends JDiscHttpRequestHandler { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java index afb491fa293..53d126ebc22 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java @@ -4,7 +4,7 @@ package ai.vespa.reindexing; import ai.vespa.reindexing.Reindexer.Cluster; import ai.vespa.reindexing.Reindexing.Trigger; import ai.vespa.reindexing.ReindexingCurator.ReindexingLockException; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.component.AbstractComponent; diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java index 7816686221b..08b3c95a543 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java @@ -3,7 +3,7 @@ package ai.vespa.reindexing.http; import ai.vespa.reindexing.Reindexing; import ai.vespa.reindexing.ReindexingCurator; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index acd00eb9817..9c33045e921 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -113,6 +113,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean experimentalSdParsing() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableBitVectors() { return false; } @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } + @ModelFeatureFlag(owners = {"tokle"}) default boolean enableProxyProtocolMixedMode() { return true; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ @@ -134,9 +135,6 @@ public interface ModelContext { default Optional<AthenzDomain> athenzDomain() { return Optional.empty(); } - // applicationRoles is no longer in use and should be removed. Replaced by AwsEnvironment. Remove after 7.458 - default Optional<ApplicationRoles> applicationRoles() { return Optional.empty(); } - default Quota quota() { return Quota.unlimited(); } default List<TenantSecretStore> tenantSecretStores() { return List.of(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index e134b8f53ac..11c55521100 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -8,7 +8,7 @@ import com.yahoo.path.Path; import java.nio.ByteBuffer; import java.util.Objects; -public class DistributableResource { +public class DistributableResource implements Comparable <DistributableResource> { public enum PathType { FILE, URI, BLOB } @@ -87,4 +87,8 @@ public class DistributableResource { return "resource '" + name + " of type '" + pathType + "' with ref '" + fileReference + "'"; } + @Override + public int compareTo(DistributableResource o) { + return name.compareTo(o.getName()); + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java index a1299c12307..5e3ef39f360 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -2,10 +2,11 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.FileRegistry; - +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class LargeRankExpressions { private final Map<String, RankExpressionBody> expressions = new ConcurrentHashMap<>(); @@ -36,6 +37,12 @@ public class LargeRankExpressions { } public int limit() { return limit; } + /** Returns a read-only list of ranking constants ordered by name */ + public Collection<RankExpressionBody> expressions() { + return expressions.values().stream().sorted().collect(Collectors.toUnmodifiableList()); + } + + // Note: Use by integration tests in internal repo /** Returns a read-only map of the ranking constants in this indexed by name */ public Map<String, RankExpressionBody> asMap() { return Collections.unmodifiableMap(expressions); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 9c802075462..e1fe795d2b1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -24,9 +24,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.Arguments; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import java.io.File; import java.io.IOException; import java.io.Reader; import java.io.Serializable; @@ -116,7 +116,7 @@ public class RankProfile implements Cloneable { // This cache must be invalidated every time modifications are done to 'functions'. private CachedFunctions allFunctionsCached = null; - private Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private Map<Reference, Input> inputs = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -764,33 +764,31 @@ public class RankProfile implements Cloneable { * All inputs must either be declared through this or in query profile types, * otherwise they are assumes to be scalars. */ - public void addInput(Reference reference, TensorType declaredType) { + public void addInput(Reference reference, Input input) { if (inputs.containsKey(reference)) { - TensorType hadType = inputs().get(reference); - if (! declaredType.equals(hadType)) - throw new IllegalArgumentException("Duplicate input '" + name + "' declared with both type " + - hadType + " and " + declaredType); + Input existing = inputs().get(reference); + if (! input.equals(existing)) + throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing"); } - inputs.put(reference, declaredType); + inputs.put(reference, input); } /** Returns the inputs of this, which also includes all inputs of the parents of this. */ // This is less restrictive than most other constructs in allowing inputs to be defined in all parent profiles // because inputs are tied closer to functions than the profile itself. - public Map<Reference, TensorType> inputs() { + public Map<Reference, Input> inputs() { if (inputs.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(inputs); // Combine - Map<Reference, TensorType> allInputs = new LinkedHashMap<>(); + Map<Reference, Input> allInputs = new LinkedHashMap<>(); for (var inheritedProfile : inherited()) { for (var input : inheritedProfile.inputs().entrySet()) { - TensorType existingType = allInputs.get(input.getKey()); - if (existingType != null && ! existingType.equals(input.getValue())) + Input existing = allInputs.get(input.getKey()); + if (existing != null && ! existing.equals(input.getValue())) throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + - input.getValue() + ", with type " + input.getValue() + "" + - " but this input is already defined with type " + existingType + - " in another profile this inherits"); + input.getValue() + ", but this input is already defined as " + + existing + " in another profile this inherits"); allInputs.put(input.getKey(), input.getValue()); } } @@ -1050,7 +1048,8 @@ public class RankProfile implements Cloneable { public MapEvaluationTypeContext typeContext() { return typeContext(new QueryProfileRegistry()); } private Map<Reference, TensorType> featureTypes() { - Map<Reference, TensorType> featureTypes = new HashMap<>(inputs()); + Map<Reference, TensorType> featureTypes = inputs().values().stream() + .collect(Collectors.toMap(input -> input.name(), input -> input.type())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); return featureTypes; @@ -1070,7 +1069,7 @@ public class RankProfile implements Cloneable { TensorType type = field.getType().asTensorType(); Optional<Reference> feature = Reference.simple(field.getName()); if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue; - if (featureTypes.containsKey(feature)) continue; // Explicit feature types (from inputs) overrides + if (featureTypes.containsKey(feature.get())) continue; // Explicit feature types (from inputs) overrides TensorType existingType = context.getType(feature.get()); if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) @@ -1392,6 +1391,46 @@ public class RankProfile implements Cloneable { } + public static final class Input { + + private final Reference name; + private final TensorType type; + private final Optional<Tensor> defaultValue; + + public Input(Reference name, TensorType type, Optional<Tensor> defaultValue) { + this.name = name; + this.type = type; + this.defaultValue = defaultValue; + } + + public Reference name() { return name; } + public TensorType type() { return type; } + public Optional<Tensor> defaultValue() { return defaultValue; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Input)) return false; + Input other = (Input)o; + if ( ! other.name().equals(this.name())) return false; + if ( ! other.type().equals(this.type())) return false; + if ( ! other.defaultValue().equals(this.defaultValue())) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, defaultValue); + } + + @Override + public String toString() { + return "input '" + name + "' " + type + + (defaultValue().isPresent() ? ":" + defaultValue.get().toAbbreviatedString() : ""); + } + + } + private static class CachedFunctions { private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index cd94b4a7f6e..4c9e65f16c8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -153,7 +153,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } public void getConfig(RankingExpressionsConfig.Builder builder) { - largeRankExpressions.asMap().values().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + largeRankExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); } public void getConfig(RankingConstantsConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 3c14a2b9c63..77245da5ddd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -145,7 +145,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); private final Map<String, String> attributeTypes; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; private final Set<String> filterFields = new java.util.LinkedHashSet<>(); private final String rankprofileName; @@ -426,10 +426,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } - for (Map.Entry<Reference, TensorType> input : inputs.entrySet()) { - if (FeatureNames.isQueryFeature(input.getKey())) - properties.add(new Pair<>("vespa.type.query." + input.getKey().arguments().expressions().get(0), - input.getValue().toString())); + for (var input : inputs.values()) { + if (FeatureNames.isQueryFeature(input.name())) { + properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), + input.type().toString())); + if (input.defaultValue().isPresent()) + properties.add(new Pair<>(input.name().toString(), + input.type().rank() == 0 ? + String.valueOf(input.defaultValue().get().asDouble()) : + input.defaultValue().get().toString(false, false))); + } } if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); distributeLargeExpressionsAsFiles(properties, largeRankExpressions); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java index eeb3a97eda9..0c1e5a76a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -15,6 +16,7 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; /** * Information about a schema. @@ -100,7 +102,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } schemaBuilder.rankprofile(rankProfileConfig); @@ -113,7 +115,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { this.name = profile.name(); @@ -125,7 +127,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } - public Map<Reference, TensorType> inputs() { return inputs; } + public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 118945369d3..63a120f7c7b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; +import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; import com.yahoo.searchdefinition.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; @@ -52,7 +53,7 @@ class ParsedRankProfile extends ParsedBlock { private final Map<String, String> fieldsRankType = new LinkedHashMap<>(); private final Map<String, List<String>> rankProperties = new LinkedHashMap<>(); private final Map<String, Value> constants = new LinkedHashMap<>(); - private final Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private final Map<Reference, RankProfile.Input> inputs = new LinkedHashMap<>(); ParsedRankProfile(String name) { super(name, "rank-profile"); @@ -83,7 +84,7 @@ class ParsedRankProfile extends ParsedBlock { Map<String, String> getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } Map<String, List<String>> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } Map<String, Value> getConstants() { return Collections.unmodifiableMap(constants); } - Map<Reference, TensorType> getInputs() { return Collections.unmodifiableMap(inputs); } + Map<Reference, RankProfile.Input> getInputs() { return Collections.unmodifiableMap(inputs); } Optional<String> getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } @@ -110,9 +111,9 @@ class ParsedRankProfile extends ParsedBlock { constants.put(name, value); } - void addInput(Reference name, TensorType type) { + void addInput(Reference name, RankProfile.Input input) { verifyThat(! inputs.containsKey(name), "already has input", name); - inputs.put(name, type); + inputs.put(name, input); } void addFieldRankFilter(String field, boolean filter) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 9de6a11ce44..fb7e67f2aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -97,7 +97,8 @@ public class RankProfileTypeSettingsProcessor extends Processor { private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) { for (RankProfile profile : rankProfileRegistry.all()) { if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence - profile.addInput(queryFeature, queryFeatureType); + profile.addInput(queryFeature, + new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index ed943317a1f..b9b7f122d63 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -153,8 +153,9 @@ public class RankSetupValidator extends Validator { // Assist verify-ranksetup in finding the actual ONNX model files writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().onnxModels().asMap().values()); - writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().asMap().values()); + writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().expressions()); + config.sort(String::compareTo); String configContent = config.isEmpty() ? "" : StringUtilities.implodeMultiline(config); IOUtils.writeFile(dir + "verify-ranksetup.cfg", configContent, false); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index e3aa5bd517f..f55fb547bb0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -25,16 +25,17 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final boolean enforceClientAuth; private final boolean enforceHandshakeClientAuth; private final Collection<String> tlsCiphersOverride; + private final boolean enableProxyProtocolMixedMode; /** * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ public static HostedSslConnectorFactory withProvidedCertificate( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth, - Collection<String> tlsCiphersOverride) { + Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth); - return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride); + return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, enableProxyProtocolMixedMode); } /** @@ -42,25 +43,28 @@ public class HostedSslConnectorFactory extends ConnectorFactory { */ public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates, - Collection<String> tlsCiphersOverride) { + Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false); - return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride); + return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode); } /** * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration). */ - public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, Collection<String> tlsCiphersOverride) { - return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride); + public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, Collection<String> tlsCiphersOverride, + boolean enableProxyProtocolMixedMode) { + return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride, enableProxyProtocolMixedMode); } private HostedSslConnectorFactory(SslProvider sslProvider, boolean enforceClientAuth, - boolean enforceHandshakeClientAuth, Collection<String> tlsCiphersOverride) { + boolean enforceHandshakeClientAuth, Collection<String> tlsCiphersOverride, + boolean enableProxyProtocolMixedMode) { super(new Builder("tls4443", 4443).sslProvider(sslProvider)); this.enforceClientAuth = enforceClientAuth; this.enforceHandshakeClientAuth = enforceHandshakeClientAuth; this.tlsCiphersOverride = tlsCiphersOverride; + this.enableProxyProtocolMixedMode = enableProxyProtocolMixedMode; } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( @@ -94,7 +98,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { } connectorBuilder - .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(true)) + .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(enableProxyProtocolMixedMode)) .idleTimeout(Duration.ofSeconds(30).toSeconds()) .maxConnectionLife(Duration.ofSeconds(45).toSeconds()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index ff1a4b6cc5f..170e22b6164 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -465,6 +465,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // If the deployment contains certificate/private key reference, setup TLS port HostedSslConnectorFactory connectorFactory; Collection<String> tlsCiphersOverride = deployState.getProperties().tlsCiphersOverride(); + boolean proxyProtocolMixedMode = deployState.getProperties().featureFlags().enableProxyProtocolMixedMode(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { @@ -480,11 +481,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore( - serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState), tlsCiphersOverride) + serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState), tlsCiphersOverride, proxyProtocolMixedMode) : HostedSslConnectorFactory.withProvidedCertificate( - serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride); + serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode); } else { - connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride); + connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride, proxyProtocolMixedMode); } cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); server.addConnector(connectorFactory); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 8cbd94a8a49..edd269559ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -215,7 +215,8 @@ public class ConvertedModel { for (ImportedMlFunction outputFunction : model.outputExpressions()) { ExpressionFunction expression = asExpressionFunction(outputFunction); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference name = Reference.fromIdentifier(input.getKey()); + profile.addInput(name, new RankProfile.Input(name, input.getValue(), Optional.empty())); } addExpression(expression, expression.getName(), constantsReplacedByFunctions, store, profile, queryProfiles, expressions); @@ -283,7 +284,8 @@ public class ConvertedModel { String name = output.getFirst(); ExpressionFunction expression = output.getSecond(); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference inputName = Reference.fromIdentifier(input.getKey()); + profile.addInput(inputName, new RankProfile.Input(inputName, input.getValue(), Optional.empty())); } TensorType type = expression.getBody().type(profile.typeContext()); if (type != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index d7ce64d1d32..1141af6d79d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -120,7 +120,7 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } docDbBuilder.rankprofile(rankProfileConfig); diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index d2d27a7a2d4..6ce261ebb8c 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1,9 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Duplicate Schema parser - work in progress -// @author arnej27959 -// NOTE: When grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf - options { UNICODE_INPUT = true; CACHE_TOKENS = false; @@ -43,7 +39,10 @@ import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorAddress; import java.util.Optional; import java.util.Map; @@ -55,6 +54,8 @@ import java.util.logging.Level; /** * The schema parser * + * NOTE: When this grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf + * * @author bratseth */ public class IntermediateParser { @@ -62,9 +63,7 @@ public class IntermediateParser { private DeployLogger deployLogger; private ModelContext.Properties properties; - /** - * Creates a parser - */ + /** Creates a parser. */ public IntermediateParser(SimpleCharStream stream, DeployLogger deployLogger, ModelContext.Properties properties) @@ -266,7 +265,7 @@ TOKEN : | < FASTRANK: "fast-rank" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > -| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < COMPRESSION: "compression" > @@ -303,6 +302,7 @@ TOKEN : | < ENABLE_BM25: "enable-bm25" > | < HNSW: "hnsw" > | < MAXLINKSPERNODE: "max-links-per-node" > +| < DOUBLE_KEYWORD: "double" > | < DISTANCEMETRIC: "distance-metric" > | < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < MULTITHREADEDINDEXING: "multi-threaded-indexing" > @@ -340,7 +340,8 @@ TOKEN : | < URI: "uri" > | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* > | < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* > -| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" > | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > | < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ > | < INTEGER: ("-")? (["0"-"9"])+ > @@ -352,8 +353,8 @@ TOKEN : | < LESSTHAN: "<" > | < GREATERTHAN: ">" > | < VARIABLE: "$" <IDENTIFIER> > -| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > -| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > } // Declare a special skip token for comments. @@ -1850,7 +1851,7 @@ void rankProfileItem(ParsedRankProfile profile) : { } } /** - * This rule consumes an inherits statement of a rank-profile. + * Consumes an inherits statement of a rank-profile. * * @param profile the profile to modify */ @@ -2067,10 +2068,35 @@ void inputs(ParsedRankProfile profile) : } { <INPUTS> <LBRACE> (<NL>)* - ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) * + ( input(profile) (<NL>)*) * <RBRACE> } +void input(ParsedRankProfile profile) : +{ + Reference reference; + TensorType type; + Tensor defaultValue = null; +} +{ + reference = queryFeature() type = inputType(reference) ( <COLON> (<NL>)* defaultValue = tensorValue(type) )? + { profile.addInput(reference, new RankProfile.Input(reference, type, Optional.ofNullable(defaultValue))); } +} + +TensorType inputType(Reference reference) : +{ + TensorType type; + +} +{ + ( + ( type = tensorType("Type of " + reference) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } +} + Reference queryFeature() : { String argument; @@ -2243,7 +2269,7 @@ void approximateThreshold(ParsedRankProfile profile) : } /** - * This rule consumes a rank-properties block of a rank profile. There + * Consumes a rank-properties block of a rank profile. There * is a little trick within this rule to allow the final rank property * to skip the terminating newline token. * @@ -2256,7 +2282,7 @@ void rankProperties(ParsedRankProfile profile) : { } } /** - * This rule consumes a single rank property pair for a rank profile. + * Consumes a single rank property pair for a rank profile. * * @param profile the rank profile to modify */ @@ -2270,25 +2296,28 @@ void rankProperty(ParsedRankProfile profile) : } /** - * This rule consumes a single rank property for a rank-properties block. + * Consumes a single rank property for a rank-properties block. * - * @return The token image of the consumed item. + * @return the token image of the consumed item */ String rankPropertyItem() : { - String image, ret = ""; + String image = null; + String ret = ""; + Token dToken = null; } { - ( ( image = identifierWithDash() { ret += image; } + ( ( image = identifierWithDash() { ret += image; } + | dToken = <DOUBLE> { ret += dToken.image; } | image = quotedString() { ret += image; } | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ ) { return ret; } } /** - * This rule consumes a field-weight statement of a rank profile. + * Consumes a field-weight statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldWeight(ParsedRankProfile profile) : { @@ -2301,9 +2330,9 @@ void fieldWeight(ParsedRankProfile profile) : } /** - * This rule consumes a rank-type statement of a rank profile. + * Consumes a rank-type statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankType(ParsedRankProfile profile) : { @@ -2316,9 +2345,9 @@ void fieldRankType(ParsedRankProfile profile) : } /** - * This rule consumes a rank filter statement of a rank profile. + * Consumes a rank filter statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankFilter(ParsedRankProfile profile) : { @@ -2330,7 +2359,7 @@ void fieldRankFilter(ParsedRankProfile profile) : } /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinSize() : { @@ -2343,7 +2372,7 @@ void rankDegradationBinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinLow() : { @@ -2354,9 +2383,8 @@ void rankDegradationBinLow() : { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); } } - /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationPosbinSize() : { @@ -2369,7 +2397,7 @@ void rankDegradationPosbinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationItem() : { } { @@ -2379,7 +2407,7 @@ void rankDegradationItem() : { } } /** - * This rule consumes a rank-degradation statement of a rank profile. + * Consumes a rank-degradation statement of a rank profile. */ void rankDegradation() : { @@ -2408,10 +2436,10 @@ void constants(ParsedRankProfile profile) : void constantValue(ParsedRankProfile profile, String name) : { - String value; + Token value; } { - <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); } } void constantTensor(ParsedRankProfile profile, String name) : @@ -2421,7 +2449,7 @@ void constantTensor(ParsedRankProfile profile, String name) : } { <LBRACE> (<NL>)* - (( tensorString = tensorValue() | + (( tensorString = tensorValuePrefixedByValue() | tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE> { if (tensorType != null) { @@ -2437,7 +2465,145 @@ String constantTensorErrorMessage(String rankProfileName, String constantTensorN { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; } } -String tensorValue() : +/** + * Parses a tensor written in a tensor literal form, + * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form + */ +Tensor tensorValue(TensorType type) : +{ + Tensor.Builder builder = Tensor.Builder.of(type); + Number doubleValue = null; +} +{ + ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = consumeNumber() ) + { + if (doubleValue != null) { + if (type.rank() > 0) + throw new IllegalArgumentException("A tensor of type " + type + " cannot be a number"); + builder.cell(doubleValue.doubleValue()); + } + return builder.build(); + } +} + +/** A mapped or mixed tensor value. */ +void mappedTensorValue(Tensor.Builder builder) : {} +{ + "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}" +} + + +void mappedTensorBlock(Tensor.Builder builder) : +{ + TensorAddress mappedAddress; +} +{ + mappedAddress = tensorAddress(builder.type().mappedSubtype()) <COLON> (<NL>)* + ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) ) +} + +void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + MixedTensor.BoundBuilder boundBuilder = (MixedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.block(sparseAddress, arrayValues); + } +} + +void indexedTensorValues(Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + IndexedTensor.BoundBuilder boundBuilder = (IndexedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.fill(arrayValues); + } +} + +/** Tensor array values. Using sub-bracketing for multiple dimensions is optional and therefore ignored here. */ +void arrayTensorValues(List<Double> values) : {} +{ + "[" ( ( indexedTensorValue(values) | arrayTensorValues(values)) )* + ( <COMMA> (<NL>)* ( indexedTensorValue(values) | arrayTensorValues(values)) )* + "]" +} + +void indexedTensorValue(List<Double> values) : +{ + Number value; +} +{ + value = consumeNumber() + { values.add(value.doubleValue()); } +} + +void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) : +{ + double value; +} +{ + value = tensorCellValue() + { builder.cell(address, value); } +} + +TensorAddress tensorAddress(TensorType type) : +{ + TensorAddress.Builder builder = new TensorAddress.Builder(type); + String label; +} +{ + ( + label = tensorAddressLabel() { builder.add(label); } + | + ( "{" ( tensorAddressElement(builder) )* ( <COMMA> tensorAddressElement(builder) )* "}" ) + ) + { return builder.build(); } +} + +void tensorAddressElement(TensorAddress.Builder builder) : +{ + String dimension; + String label; +} +{ + dimension = identifier() <COLON> (<NL>)* label = tensorAddressLabel() + { builder.add(dimension, label); } +} + +String tensorAddressLabel() : +{ + String label; +} +{ + ( label = identifier() | label = quotedString() ) + { return label; } +} + +double tensorCellValue() : +{ + Number value; +} +{ + value = consumeNumber() + { return value.doubleValue(); } +} + +/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */ +String tensorValuePrefixedByValue() : { String tensor; } @@ -2462,7 +2628,7 @@ TensorType tensorType(String errorMessage) : String tensorTypeString; } { - ( <TENSOR_TYPE> ) { tensorTypeString = token.image; } + <TENSOR_TYPE> { tensorTypeString = token.image; } { TensorType tensorType; try { @@ -2555,7 +2721,7 @@ String identifier() : { } | <DIRECT> | <DOCUMENT> | <DOCUMENTSUMMARY> - | <DOUBLE> + | <DOUBLE_KEYWORD> | <DYNAMIC> | <ENABLEBITVECTORS> | <ENABLEONLYBITVECTOR> @@ -2683,39 +2849,28 @@ String string() : { } * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can * contain anything but a double quote. * - * @return The unquoted token image. + * @return the unquoted token image */ String quotedString() : { } { - <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); } + ( <DOUBLEQUOTEDSTRING> | <SINGLEQUOTEDSTRING> ) + { return token.image.substring(1, token.image.length() - 1); } } -/** - * This rule consumes a boolean value. - * - * @return The consumed boolean value. - */ +/** A boolean value. */ Boolean bool() : { } { ( ( <ON> | <TRUE> ) { return true; } | ( <OFF> | <FALSE> ) { return false; } ) } -/** - * This rule consumes an integer token and returns its numeric value. - * - * @return The consumed integer value. - */ +/** Consumes an integer token and returns its numeric value. */ int integer() : { } { <INTEGER> { return Integer.parseInt(token.image); } } -/** - * This rule consumes a long or integer token and returns its numeric value. - * - * @return The consumed long value. - */ +/** Consumes a long or integer token and returns its numeric value. */ long consumeLong() : { } { ( <INTEGER> { return Long.parseLong(token.image); } | @@ -2723,11 +2878,7 @@ long consumeLong() : { } ) } -/** - * This rule consumes a floating-point token and returns its numeric value. - * - * @return The consumed value. - */ +/** Consumes a floating-point token and returns its numeric value. */ double consumeFloat() : { } { <DOUBLE> { return Double.valueOf(token.image); } @@ -2741,9 +2892,7 @@ Number consumeNumber() : (num = consumeFloat() | num = consumeLong()) { return num; } } -/** - * This rule consumes an opening brace with leading and trailing newline tokens. - */ +/** Consumes an opening brace with leading and trailing newline tokens. */ void lbrace() : { } { (<NL>)* <LBRACE> (<NL>)* diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 0ff9513885f..de2ce3d6938 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -202,6 +202,7 @@ TOKEN : | < INDEXING: "indexing" > | < SUMMARYTO: "summary-to" > | < DOCUMENTSUMMARY: "document-summary" > +| <DOUBLE_KEYWORD: "double" > | < RANKTYPE: "rank-type" > | < WEIGHT: "weight" > | < TYPE: "type" > @@ -296,7 +297,7 @@ TOKEN : | < PAGED: "paged" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > -| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < COMPRESSION: "compression" > @@ -2196,15 +2197,36 @@ void secondPhaseItem(RankProfile profile) : } /** Consumes an inputs block of a rank profile. */ -void inputs(RankProfile profile) : +void inputs(RankProfile profile) : {} +{ + <INPUTS> <LBRACE> (<NL>)* + ( input(profile) (<NL>)*) * + <RBRACE> +} + +void input(RankProfile profile) : { Reference reference; TensorType type; } { - <INPUTS> <LBRACE> (<NL>)* - ( reference = queryFeature() type = tensorType("Type of " + reference) { profile.addInput(reference, type); } (<NL>)*) * - <RBRACE> + reference = queryFeature() type = inputType(reference) + { + profile.addInput(reference, new RankProfile.Input(reference, type, Optional.empty())); + } +} + +TensorType inputType(Reference reference) : +{ + TensorType type; +} +{ + ( + ( type = tensorType("Type of " + reference) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } } Reference queryFeature() : @@ -2408,10 +2430,13 @@ void rankProperty(RankProfile profile) : */ String rankPropertyItem() : { - String image, ret = ""; + String image; + String ret = ""; + Token dToken; } { - ( ( image = identifierWithDash() { ret += image; } + ( ( image = identifierWithDash() { ret += image; } + | dToken = <DOUBLE> { ret += dToken.image; } | image = quotedString() { ret += image; } | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ ) { return ret; } @@ -2542,10 +2567,10 @@ void constants(RankProfile profile) : void constantValue(RankProfile profile, String name) : { - String value; + Token value; } { - <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); } } void constantTensor(RankProfile profile, String name) : @@ -2596,7 +2621,7 @@ TensorType tensorType(String errorMessage) : String tensorTypeString; } { - ( <TENSOR_TYPE> ) { tensorTypeString = token.image; } + <TENSOR_TYPE> { tensorTypeString = token.image; } { TensorType tensorType; try { @@ -2693,7 +2718,7 @@ String identifier() : { } | <DIRECT> | <DOCUMENT> | <DOCUMENTSUMMARY> - | <DOUBLE> + | <DOUBLE_KEYWORD> | <DYNAMIC> | <ENABLEBITVECTORS> | <ENABLEONLYBITVECTOR> diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd index 9069f59bbe3..aed80f77e6f 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd @@ -77,7 +77,7 @@ search neuralnet { query(W_1) tensor(hidden[9],out[9]) query(b_1) tensor(out[9]) query(W_out) tensor(out[9]) - query(b_out) tensor(out[1]) + query(b_out) tensor(out[1]):[1.0] } } diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg index f5134dd15f9..1e05fe77324 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg @@ -11,6 +11,8 @@ rankprofile[].fef.property[].name "vespa.type.query.W_out" rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.b_out" rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" rankprofile[].name "unranked" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "value(0)" @@ -69,18 +71,20 @@ rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScrip rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeRank" -rankprofile[].fef.property[].name "vespa.type.query.b_out" -rankprofile[].fef.property[].value "tensor(out[1])" -rankprofile[].fef.property[].name "vespa.type.query.W_out" -rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_0" +rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" rankprofile[].fef.property[].name "vespa.type.query.b_0" rankprofile[].fef.property[].value "tensor(hidden[9])" -rankprofile[].fef.property[].name "vespa.type.query.b_1" -rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.W_1" rankprofile[].fef.property[].value "tensor(hidden[9],out[9])" -rankprofile[].fef.property[].name "vespa.type.query.W_0" -rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_1" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_out" +rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" +rankprofile[].fef.property[].name "vespa.type.query.W_out" +rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].name "neuralNetworkProfile" rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScript" rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)" @@ -172,15 +176,17 @@ rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(layer_out)" rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" rankprofile[].fef.property[].value "2000" -rankprofile[].fef.property[].name "vespa.type.query.b_out" -rankprofile[].fef.property[].value "tensor(out[1])" -rankprofile[].fef.property[].name "vespa.type.query.W_out" -rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_0" +rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" rankprofile[].fef.property[].name "vespa.type.query.b_0" rankprofile[].fef.property[].value "tensor(hidden[9])" -rankprofile[].fef.property[].name "vespa.type.query.b_1" -rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.W_1" rankprofile[].fef.property[].value "tensor(hidden[9],out[9])" -rankprofile[].fef.property[].name "vespa.type.query.W_0" -rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_1" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.W_out" +rankprofile[].fef.property[].value "tensor(out[9])" +rankprofile[].fef.property[].name "vespa.type.query.b_out" +rankprofile[].fef.property[].value "tensor(out[1])" +rankprofile[].fef.property[].name "query(b_out)" +rankprofile[].fef.property[].value "{{out:0}:1.0}" diff --git a/config-model/src/test/derived/rankproperties/rank-profiles.cfg b/config-model/src/test/derived/rankproperties/rank-profiles.cfg index 47a0438a323..3ca44288282 100644 --- a/config-model/src/test/derived/rankproperties/rank-profiles.cfg +++ b/config-model/src/test/derived/rankproperties/rank-profiles.cfg @@ -1,6 +1,8 @@ rankprofile[].name "default" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" @@ -21,8 +23,10 @@ rankprofile[].fef.property[].value "0" rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" rankprofile[].fef.property[].value "true" rankprofile[].name "child" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" diff --git a/config-model/src/test/derived/rankproperties/rankproperties.sd b/config-model/src/test/derived/rankproperties/rankproperties.sd index 24977ae645f..db259f1daec 100644 --- a/config-model/src/test/derived/rankproperties/rankproperties.sd +++ b/config-model/src/test/derived/rankproperties/rankproperties.sd @@ -28,7 +28,8 @@ search rankproperties { expression: match } rank-properties { - $test:"foo" + $test1:"foo" + query(test2): 12.3 #$weight:1 } } diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd index e26358fef5a..339d147dfd0 100644 --- a/config-model/src/test/examples/rankpropvars.sd +++ b/config-model/src/test/examples/rankpropvars.sd @@ -55,13 +55,11 @@ document music { } field artist type string { - ## index-to: a indexing: index | summary } field year type int { indexing: attribute | summary - ## index-to: y } field url type uri {} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java index 86ccb816c10..0f29d2dda40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java @@ -10,8 +10,10 @@ import java.io.IOException; * @author bratseth */ public class RankPropertiesTestCase extends AbstractExportingTestCase { + @Test public void testRankProperties() throws IOException, ParseException { assertCorrectDeriving("rankproperties"); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index bcb197ed540..0b7dbf2eb21 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -435,7 +435,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } private VespaModel getIndexedSearchVespaModel(String xml) { - List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist"); + List<String> sds = generateSchemas("music", "album", "artist"); return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); } @@ -501,4 +501,55 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } } + public static String generateSchema(String name, String field1, String field2) { + return "schema " + name + " {" + + " document " + name + " {" + + " field " + field1 + " type string {\n" + + " indexing: index | summary\n" + + " summary: dynamic\n" + + " }\n" + + " field " + field2 + " type int {\n" + + " indexing: attribute | summary\n" + + " attribute: fast-access\n" + + " }\n" + + " field " + field2 + "_nfa type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " rank-profile staticrank inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }" + + " }" + + " rank-profile summaryfeatures inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " summary-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" + + " }" + + " rank-profile rankfeatures {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " rank-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " }" + + " }" + + "}"; + } + + public static List<String> generateSchemas(String ... sdNames) { + return generateSchemas(Arrays.asList(sdNames)); + } + + public static List<String> generateSchemas(List<String> sdNames) { + List<String> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2))); + i = i + 2; + } + return sds; + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 3cff04198bd..671a9f6660d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; @@ -16,11 +15,8 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; -import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; -import java.util.ArrayList; import java.util.List; import java.util.Arrays; import java.util.Map; @@ -35,92 +31,9 @@ public class DocumentDatabaseTestCase { private static final double SMALL = 0.00000000000001; - private static final String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" + - "<hosts> " + - " <host name='foo'>" + - " <alias>node0</alias>" + - " </host>" + - "</hosts>"; - - private String createVespaServices(List<String> sds, String mode) { - List<DocType> nameAndModes = new ArrayList<>(sds.size()); - for (String sd : sds) { - nameAndModes.add(DocType.create(sd, mode)); - } - return createVespaServicesXml(nameAndModes, ""); - } - private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) { - StringBuilder retval = new StringBuilder(); - retval.append("" + - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services version='1.0'>\n" + - "<admin version='2.0'>\n" + - " <adminserver hostalias='node0' />\n" + - "</admin>\n" + - "<container version='1.0'>\n" + - " <nodes>\n" + - " <node hostalias='node0'/>\n" + - " </nodes>\n" + - " <search/>\n" + - "</container>\n" + - "<content version='1.0' id='test'>\n" + - " <redundancy>1</redundancy>\n"); - retval.append(DocType.listToXml(nameAndModes)); - retval.append( - " <engine>\n" + - " <proton>\n" + - " <tuning>\n" + - " <searchnode>\n" + - xmlTuning + - " </searchnode>\n" + - " </tuning\n>" + - " </proton\n>" + - " </engine\n>" + - " <nodes>\n" + - " <node hostalias='node0' distribution-key='0'/>\n" + - " </nodes>\n" + - " </content>\n" + - "</services>\n"); - return retval.toString(); - } - - private ProtonConfig getProtonCfg(ContentSearchCluster cluster) { - ProtonConfig.Builder pb = new ProtonConfig.Builder(); - cluster.getConfig(pb); - return new ProtonConfig(pb); - } - - private void assertSingleSD(String mode) { - final List<String> sds = Arrays.asList("type1"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(); - IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); - ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); - String type1Id = "test/search/cluster.test/type1"; - ProtonConfig proton = getProtonCfg(contentSearchCluster); - assertEquals(1, proton.documentdb().size()); - assertEquals("type1", proton.documentdb(0).inputdoctypename()); - assertEquals(type1Id, proton.documentdb(0).configid()); - } - @Test public void requireThatWeCanHaveOneSDForIndexedMode() { - assertSingleSD("index"); - } - - private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { - return createModel(nameAndModes, xmlTuning, null); - } - - private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { - List<String> sds = new ArrayList<>(nameAndModes.size()); - for (DocType nameAndMode : nameAndModes) { - sds.add(nameAndMode.getType()); - } - var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), - ApplicationPackageUtils.generateSchemas(sds)); - return builder != null ? creator.create(builder) : creator.create(); + new SchemaTester().assertSingleSD("index"); } @Test @@ -178,9 +91,10 @@ public class DocumentDatabaseTestCase { if (featureFlagConcurrency != null) { properties.setFeedConcurrency(featureFlagConcurrency); } - VespaModel model = createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); + var tester = new SchemaTester(); + VespaModel model = tester.createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(global, proton.feeding().concurrency(), SMALL); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { @@ -190,11 +104,12 @@ public class DocumentDatabaseTestCase { @Test public void requireThatModeIsSet() { - VespaModel model = createModel(Arrays.asList(DocType.create("a", "index"), - DocType.create("b", "streaming"), - DocType.create("c", "store-only")), ""); + var tester = new SchemaTester(); + VespaModel model = tester.createModel(Arrays.asList(DocType.create("a", "index"), + DocType.create("b", "streaming"), + DocType.create("c", "store-only")), ""); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode()); assertEquals("a", proton.documentdb(0).inputdoctypename()); @@ -205,10 +120,11 @@ public class DocumentDatabaseTestCase { } private void verifyInitialDocumentCount(List<DocType> nameAndModes, String xmlTuning, List<Long> local) { + var tester = new SchemaTester(); assertEquals(nameAndModes.size(), local.size()); - VespaModel model = createModel(nameAndModes, xmlTuning); + VespaModel model = tester.createModel(nameAndModes, xmlTuning); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { assertEquals(local.get(i).longValue(), proton.documentdb(i).allocation().initialnumdocs()); @@ -240,14 +156,14 @@ public class DocumentDatabaseTestCase { assertEquals(attributeField, acfg.attribute(0).name()); assertEquals(attributeField+"_nfa", acfg.attribute(1).name()); RankProfilesConfig rcfg = model.getConfig(RankProfilesConfig.class, configId); - assertEquals(7, rcfg.rankprofile().size()); + assertEquals(6, rcfg.rankprofile().size()); } @Test public void testMultipleSchemas() { List<String> sds = List.of("type1", "type2", "type3"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"), - ApplicationPackageUtils.generateSchemas(sds)).create(); + var tester = new SchemaTester(); + var model = tester.createModel(sds); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); String type1Id = "test/search/cluster.test/type1"; @@ -255,7 +171,7 @@ public class DocumentDatabaseTestCase { String type3Id = "test/search/cluster.test/type3"; { assertEquals(3, indexedSearchCluster.getDocumentDbs().size()); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals("type1", proton.documentdb(0).inputdoctypename()); assertEquals(type1Id, proton.documentdb(0).configid()); @@ -295,9 +211,16 @@ public class DocumentDatabaseTestCase { @Test public void requireThatRelevantConfigIsAvailableForClusterSearcher() { - List<String> schemas = Arrays.asList("type1", "type2"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - ApplicationPackageUtils.generateSchemas(schemas)).create(); + String inputsProfile = + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " }" + + " }"; + List<String> schemas = List.of("type1", "type2"); + var tester = new SchemaTester(); + VespaModel model = tester.createModelWithRankProfile(inputsProfile, schemas); String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher"; { // documentdb-info config @@ -309,13 +232,13 @@ public class DocumentDatabaseTestCase { assertEquals("type1", db.name()); assertEquals(7, db.rankprofile().size()); - assertRankProfile(db, 0, "default", false, false); - assertRankProfile(db, 1, "unranked", false, false); - assertRankProfile(db, 2, "staticrank", false, false); - assertRankProfile(db, 3, "summaryfeatures", true, false); - assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); - assertRankProfile(db, 5, "rankfeatures", false, true); - var inputs = assertRankProfile(db, 6, "inputs", false, false); + tester.assertRankProfile(db, 0, "default", false, false); + tester.assertRankProfile(db, 1, "unranked", false, false); + tester.assertRankProfile(db, 2, "staticrank", false, false); + tester.assertRankProfile(db, 3, "summaryfeatures", true, false); + tester.assertRankProfile(db, 4, "inheritedsummaryfeatures", true, false); + tester.assertRankProfile(db, 5, "rankfeatures", false, true); + var inputs = tester.assertRankProfile(db, 6, "inputs", false, false); assertEquals(2, inputs.input().size()); assertEquals("query(foo)", inputs.input(0).name()); @@ -326,8 +249,8 @@ public class DocumentDatabaseTestCase { assertEquals(2, db.summaryclass().size()); assertEquals("default", db.summaryclass(0).name()); assertEquals("attributeprefetch", db.summaryclass(1).name()); - assertSummaryField(db, 0, 0, "f1", "longstring", true); - assertSummaryField(db, 0, 1, "f2", "integer", false); + tester.assertSummaryField(db, 0, 0, "f1", "longstring", true); + tester.assertSummaryField(db, 0, 1, "f2", "integer", false); } { // type2 DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(1); @@ -345,89 +268,10 @@ public class DocumentDatabaseTestCase { } } - /** Schema-info should contain all schemas, independent of clusters. */ - @Test - public void requireThatSchemaInfoIsAvailable() { - List<String> schemas = Arrays.asList("type1", "type2"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - ApplicationPackageUtils.generateSchemas(schemas)).create(); - assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model); - assertSchemaInfo("container", model); - } - - private void assertSchemaInfo(String configId, VespaModel model) { - { // schema-info config - SchemaInfoConfig dcfg = model.getConfig(SchemaInfoConfig.class, configId); - assertEquals(2, dcfg.schema().size()); - - { // type1 - SchemaInfoConfig.Schema schema = dcfg.schema(0); - assertEquals("type1", schema.name()); - - assertEquals(7, schema.rankprofile().size()); - assertRankProfile(schema, 0, "default", false, false); - assertRankProfile(schema, 1, "unranked", false, false); - assertRankProfile(schema, 2, "staticrank", false, false); - assertRankProfile(schema, 3, "summaryfeatures", true, false); - assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); - assertRankProfile(schema, 5, "rankfeatures", false, true); - var inputs = assertRankProfile(schema, 6, "inputs", false, false); - - assertEquals(2, inputs.input().size()); - assertEquals("query(foo)", inputs.input(0).name()); - assertEquals("tensor<float>(x[10])", inputs.input(0).type()); - assertEquals("query(bar)", inputs.input(1).name()); - assertEquals("tensor(key{},x[1000])", inputs.input(1).type()); - - // assertEquals(2, schema.summaryclass().size()); - // assertEquals("default", schema.summaryclass(0).name()); - // assertEquals("attributeprefetch", schema.summaryclass(1).name()); - // assertSummaryField(schema, 0, 0, "f1", "longstring", true); - // assertSummaryField(schema, 0, 1, "f2", "integer", false); - } - { // type2 - SchemaInfoConfig.Schema schema = dcfg.schema(1); - assertEquals("type2", schema.name()); - } - } - } - - private DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, - int index, - String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures) { - DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index); - assertEquals(name, rankProfile.name()); - assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); - assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); - return rankProfile; - } - - private SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, - int index, - String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures) { - SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index); - assertEquals(name, rankProfile.name()); - assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); - assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); - return rankProfile; - } - - private void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, - String name, String type, boolean dynamic) { - DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); - assertEquals(name, field.name()); - assertEquals(type, field.type()); - assertEquals(dynamic, field.dynamic()); - } - private void assertDocumentDBConfigAvailableForStreaming(String mode) { - final List<String> sds = Arrays.asList("type"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(); + List<String> sds = List.of("type"); + var tester = new SchemaTester(); + var model = tester.createModelWithMode(mode, sds); DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); assertEquals(1, dcfg.documentdb().size()); @@ -452,11 +296,11 @@ public class DocumentDatabaseTestCase { Map<String, List<String>> expectedAttributesMap, DeployState.Builder builder, long expectedMaxUnCommittedMemory) { - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSchemas(sds)).create(builder); + var tester = new SchemaTester(); + var model = tester.createModelWithMode(mode, sds, builder); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); assertEquals(sds.size(), proton.documentdb().size()); for (int i = 0; i < sds.size(); i++) { assertEquals(sds.get(i), proton.documentdb(i).inputdoctypename()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java new file mode 100644 index 00000000000..cf2135aea5a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class SchemaInfoTestCase { + + /** Schema-info should contain all schemas, independent of clusters. */ + @Test + public void requireThatSchemaInfoIsAvailable() { + List.of(1.0,2.0,3.0).toArray(new Double[3]); + String inputs = + " rank-profile inputs {" + + " inputs {" + + " query(foo) tensor<float>(x[10])" + + " query(bar) tensor(key{},x[1000])" + + " query(myDouble1) double: 0.5" + + " query(myDouble2) tensor()" + + " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" + + " query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" + + " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" + + " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" + + " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" + + " }" + + " }"; + List<String> schemas = List.of("type1", "type2"); + var tester = new SchemaTester(); + var model = tester.createModelWithRankProfile(inputs, schemas); + assertSchemaInfo("container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher", model, tester); + assertSchemaInfo("container", model, tester); + } + + private void assertSchemaInfo(String configId, VespaModel model, SchemaTester tester) { + { + SchemaInfoConfig schemaInfoConfig = model.getConfig(SchemaInfoConfig.class, configId); + RankProfilesConfig rankProfilesConfig = model.getConfig(RankProfilesConfig.class, "test/search/cluster.test/type1"); + + assertEquals(2, schemaInfoConfig.schema().size()); + + { // type1 + SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(0); + assertEquals("type1", schema.name()); + + assertEquals(7, schema.rankprofile().size()); + tester.assertRankProfile(schema, 0, "default", false, false); + tester.assertRankProfile(schema, 1, "unranked", false, false); + tester.assertRankProfile(schema, 2, "staticrank", false, false); + tester.assertRankProfile(schema, 3, "summaryfeatures", true, false); + tester.assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); + tester.assertRankProfile(schema, 5, "rankfeatures", false, true); + + var schemaInfoProfile = tester.assertRankProfile(schema, 6, "inputs", false, false); + assertEquals(9, schemaInfoProfile.input().size()); + var rankProfilesProfile = rankProfilesConfig.rankprofile().get(6); + assertEquals("inputs", rankProfilesProfile.name()); + assertInput("query(foo)", "tensor<float>(x[10])", null, 0, schemaInfoProfile, rankProfilesProfile); + assertInput("query(bar)", "tensor(key{},x[1000])", null, 1, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble1)", "tensor()", "0.5", 2, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble2)", "tensor()", null, 3, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMap)", "tensor(key{})", "{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myVector)", "tensor(x[3])", "{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMatrix)", "tensor(x[2],y[3])", "{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 6, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed1)", "tensor(key{},x[2])", "{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 7, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile); + + assertEquals(2, schema.summaryclass().size()); + assertEquals("default", schema.summaryclass(0).name()); + assertEquals("attributeprefetch", schema.summaryclass(1).name()); + tester.assertSummaryField(schema, 0, 0, "f1", "longstring", true); + tester.assertSummaryField(schema, 0, 1, "f2", "integer", false); + } + { // type2 + SchemaInfoConfig.Schema schema = schemaInfoConfig.schema(1); + assertEquals("type2", schema.name()); + } + } + } + + private void assertInput(String name, String type, String defaultValue, + int index, + SchemaInfoConfig.Schema.Rankprofile schemaInfoProfile, + RankProfilesConfig.Rankprofile rankProfilesProfile) { + assertEquals(name, schemaInfoProfile.input(index).name()); + assertEquals(type, schemaInfoProfile.input(index).type()); + if (defaultValue != null) { + boolean found = false; + for (var property : rankProfilesProfile.fef().property()) { + if (property.name().equals(name)) { + assertEquals(defaultValue, property.value()); + found = true; + } + } + if ( ! found) + fail("Missing property " + name); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java new file mode 100644 index 00000000000..7a077c1a0aa --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java @@ -0,0 +1,220 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search.test; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.content.utils.DocType; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SchemaTester { + + private static final String vespaHosts = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<hosts> " + + " <host name='foo'>" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + private String createVespaServices(List<String> sds, String mode) { + List<DocType> nameAndModes = new ArrayList<>(sds.size()); + for (String sd : sds) { + nameAndModes.add(DocType.create(sd, mode)); + } + return createVespaServicesXml(nameAndModes, ""); + } + private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) { + StringBuilder retval = new StringBuilder(); + retval.append("" + + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services version='1.0'>\n" + + "<admin version='2.0'>\n" + + " <adminserver hostalias='node0' />\n" + + "</admin>\n" + + "<container version='1.0'>\n" + + " <nodes>\n" + + " <node hostalias='node0'/>\n" + + " </nodes>\n" + + " <search/>\n" + + "</container>\n" + + "<content version='1.0' id='test'>\n" + + " <redundancy>1</redundancy>\n"); + retval.append(DocType.listToXml(nameAndModes)); + retval.append( + " <engine>\n" + + " <proton>\n" + + " <tuning>\n" + + " <searchnode>\n" + + xmlTuning + + " </searchnode>\n" + + " </tuning\n>" + + " </proton\n>" + + " </engine\n>" + + " <nodes>\n" + + " <node hostalias='node0' distribution-key='0'/>\n" + + " </nodes>\n" + + " </content>\n" + + "</services>\n"); + return retval.toString(); + } + + ProtonConfig getProtonCfg(ContentSearchCluster cluster) { + ProtonConfig.Builder pb = new ProtonConfig.Builder(); + cluster.getConfig(pb); + return new ProtonConfig(pb); + } + + void assertSingleSD(String mode) { + List<String> sds = List.of("type1"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), + generateSchemas("", sds)).create(); + IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); + String type1Id = "test/search/cluster.test/type1"; + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(1, proton.documentdb().size()); + assertEquals("type1", proton.documentdb(0).inputdoctypename()); + assertEquals(type1Id, proton.documentdb(0).configid()); + } + + VespaModel createModel(List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas("", schemas)).create(); + } + + VespaModel createModelWithRankProfile(String rankProfile, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas(rankProfile, schemas)).create(); + } + + VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { + return createModel(nameAndModes, xmlTuning, null); + } + + VespaModel createModelWithMode(String mode, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, mode), + generateSchemas("", schemas)).create(); + + } + + VespaModel createModelWithMode(String mode, List<String> schemas, DeployState.Builder builder) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, mode), + generateSchemas("", schemas)).create(builder); + } + + VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { + List<String> sds = new ArrayList<>(nameAndModes.size()); + for (DocType nameAndMode : nameAndModes) { + sds.add(nameAndMode.getType()); + } + var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), + generateSchemas("", sds)); + return builder != null ? creator.create(builder) : creator.create(); + } + + public static String generateSchema(String name, String field1, String field2, String rankProfile) { + return "schema " + name + " {" + + " document " + name + " {" + + " field " + field1 + " type string {\n" + + " indexing: index | summary\n" + + " summary: dynamic\n" + + " }\n" + + " field " + field2 + " type int {\n" + + " indexing: attribute | summary\n" + + " attribute: fast-access\n" + + " }\n" + + " field " + field2 + "_nfa type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " rank-profile staticrank inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }" + + " }" + + " rank-profile summaryfeatures inherits default {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " summary-features: attribute(" + field2 + ")" + + " }" + + " rank-profile inheritedsummaryfeatures inherits summaryfeatures {" + + " }" + + " rank-profile rankfeatures {" + + " first-phase { expression: attribute(" + field2 + ") }\n" + + " rank-features: attribute(" + field2 + ")" + + " }" + + rankProfile + + "}"; + } + + public static List<String> generateSchemas(String rankProfile, String ... sdNames) { + return generateSchemas(rankProfile, Arrays.asList(sdNames)); + } + + public static List<String> generateSchemas(String rankProfile, List<String> sdNames) { + List<String> sds = new ArrayList<>(); + int i = 0; + for (String sdName : sdNames) { + sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), rankProfile)); + i = i + 2; + } + return sds; + } + + DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, + int index, + String name, + boolean hasSummaryFeatures, + boolean hasRankFeatures) { + DocumentdbInfoConfig.Documentdb.Rankprofile rankProfile = db.rankprofile(index); + assertEquals(name, rankProfile.name()); + assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); + return rankProfile; + } + + SchemaInfoConfig.Schema.Rankprofile assertRankProfile(SchemaInfoConfig.Schema schema, + int index, + String name, + boolean hasSummaryFeatures, + boolean hasRankFeatures) { + SchemaInfoConfig.Schema.Rankprofile rankProfile = schema.rankprofile(index); + assertEquals(name, rankProfile.name()); + assertEquals(hasSummaryFeatures, rankProfile.hasSummaryFeatures()); + assertEquals(hasRankFeatures, rankProfile.hasRankFeatures()); + return rankProfile; + } + + void assertSummaryField(DocumentdbInfoConfig.Documentdb db, int summaryClassIndex, int fieldIndex, + String name, String type, boolean dynamic) { + DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field = db.summaryclass(summaryClassIndex).fields(fieldIndex); + assertEquals(name, field.name()); + assertEquals(type, field.type()); + assertEquals(dynamic, field.dynamic()); + } + + void assertSummaryField(SchemaInfoConfig.Schema schema, int summaryClassIndex, int fieldIndex, + String name, String type, boolean dynamic) { + SchemaInfoConfig.Schema.Summaryclass.Fields field = schema.summaryclass(summaryClassIndex).fields(fieldIndex); + assertEquals(name, field.name()); + assertEquals(type, field.type()); + assertEquals(dynamic, field.dynamic()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index 952de2a6f40..df611deb72a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -7,6 +7,7 @@ import java.util.List; /** * For testing purposes only. + * * @author geirst */ public class ApplicationPackageUtils { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java index a0de085ef96..61b56912146 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provisioning.FlavorsConfig; import java.util.ArrayList; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java index 03c35f789b1..1f1b3dc5699 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provisioning.CloudConfig; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java index 3e75842984f..c6ace00b90c 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java @@ -35,7 +35,7 @@ public interface ZoneList extends ZoneFilter { /** Returns the ZoneIds of all zones in this list. */ default List<ZoneId> ids() { - return zones().stream().map(ZoneApi::getId).collect(Collectors.toList()); + return zones().stream().map(ZoneApi::getVirtualId).collect(Collectors.toList()); } } diff --git a/config/src/vespa/config/file_acquirer/file_acquirer.cpp b/config/src/vespa/config/file_acquirer/file_acquirer.cpp index 9cf280566e0..048e9544dab 100644 --- a/config/src/vespa/config/file_acquirer/file_acquirer.cpp +++ b/config/src/vespa/config/file_acquirer/file_acquirer.cpp @@ -5,7 +5,6 @@ #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/transport.h> -#include <vespa/vespalib/util/size_literals.h> #include <vespa/fastos/thread.h> #include <vespa/log/log.h> diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java index be98986f115..b7c4cac861d 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.configserver.flags; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.vespa.configserver.flags.db.BootstrapFlagSource; import com.yahoo.vespa.configserver.flags.db.ZooKeeperFlagSource; import com.yahoo.vespa.flags.OrderedFlagSource; diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java index ba8047d65b8..3d064636ab6 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/FlagsDbImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.configserver.flags.db; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.path.Path; import com.yahoo.vespa.configserver.flags.FlagsDb; import com.yahoo.vespa.curator.Curator; diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java index f77c0cbf932..0d640bfb640 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.configserver.flags.http; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index fbad212b85f..1042c38f995 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -207,6 +207,7 @@ public class ModelContextImpl implements ModelContext { private final boolean experimentalSdParsing; private final boolean enableBitVectors; private final Architecture adminClusterArchitecture; + private final boolean enableProxyProtocolMixedMode; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -251,6 +252,7 @@ public class ModelContextImpl implements ModelContext { this.experimentalSdParsing = flagValue(source, appId, version, Flags.EXPERIMENTAL_SD_PARSING); this.enableBitVectors = flagValue(source, appId, version, Flags.ENABLE_BIT_VECTORS); this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); + this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -297,6 +299,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean experimentalSdParsing() { return experimentalSdParsing; } @Override public boolean enableBitVectors() { return this.enableBitVectors; } @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } + @Override public boolean enableProxyProtocolMixedMode() { return enableProxyProtocolMixedMode; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-core/pom.xml b/container-core/pom.xml index 7b1b6195879..be22e5cae5c 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -19,11 +19,6 @@ <!-- COMPILE scope --> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>annotations</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-documentapi</artifactId> <version>${project.version}</version> </dependency> @@ -128,22 +123,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> </dependency> @@ -187,6 +166,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>component</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -223,6 +208,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>jdisc_core</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -235,13 +226,25 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>yolean</artifactId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>hosted-zone-api</artifactId> + <artifactId>yolean</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 00c3a1d1aae..95494190734 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -47,7 +47,7 @@ public class Coverage { protected Coverage(long docs, int nodes, boolean full, int resultSets) { this(docs, docs, nodes, resultSets, full ? FullCoverageDefinition.EXPLICITLY_FULL - : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); + : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); } private Coverage(long docs, long active, int nodes, int resultSets, FullCoverageDefinition fullReason) { @@ -168,9 +168,8 @@ public class Coverage { } /** - * An int between 0 (inclusive) and 100 (inclusive) representing how many - * percent coverage the result sets this Coverage instance contains information - * about had. + * An int between 0 (inclusive) and 100 (inclusive) representing the + * percent coverage of the result sets this instance contains information about. */ public int getResultPercentage() { if (getResultSets() == 0) { diff --git a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java index 4909f9a76eb..1e171a19b05 100644 --- a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java +++ b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java @@ -27,6 +27,8 @@ public class OsgiImpl implements Osgi { // The initial bundles are never scheduled for uninstall private final List<Bundle> initialBundles; + private final Bundle systemBundle; + // An initial bundle that is not the framework, and can hence be used to look up current bundles private final Bundle alwaysCurrentBundle; @@ -37,6 +39,7 @@ public class OsgiImpl implements Osgi { if (initialBundles.isEmpty()) throw new IllegalStateException("No initial bundles!"); + systemBundle = getSystemBundle(initialBundles, jdiscOsgi); alwaysCurrentBundle = firstNonFrameworkBundle(initialBundles); if (alwaysCurrentBundle == null) throw new IllegalStateException("The initial bundles only contained the framework bundle!"); @@ -59,20 +62,33 @@ public class OsgiImpl implements Osgi { if (bundle != null) { return resolveFromBundle(spec, bundle); } else { - return resolveFromClassPath(spec); + return resolveFromThisBundleOrSystemBundle(spec); } } + /** + * Tries to resolve the given class from this class' bundle classloader. + * If unsuccessful, resolves the class from . + */ @SuppressWarnings("unchecked") - private static Class<Object> resolveFromClassPath(BundleInstantiationSpecification spec) { + private Class<Object> resolveFromThisBundleOrSystemBundle(BundleInstantiationSpecification spec) { + log.fine(() -> "Resolving class from container-disc: " + spec.classId.getName()); try { return (Class<Object>) Class.forName(spec.classId.getName()); } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Could not create a component with id '" + spec.classId.getName() + - "'. Tried to load class directly, since no bundle was found for spec: " + spec.bundle + - ". If a bundle with the same name is installed, there is a either a version mismatch" + - " or the installed bundle's version contains a qualifier string."); + if (hasFelixFramework()) { + log.fine(() -> "Resolving class from the system bundle (jdisc core): " + spec.classId.getName()); + try { + return (Class<Object>) systemBundle.loadClass(spec.classId.getName()); + } catch (ClassNotFoundException e2) { + // empty + } + } } + throw new IllegalArgumentException( + "Could not create a component with id '" + spec.classId.getName() + "'. Tried to load class directly, " + + "since no bundle was found for spec: " + spec.bundle + ". If a bundle with the same name is installed, " + + "there is a either a version mismatch or the installed bundle's version contains a qualifier string."); } @SuppressWarnings("unchecked") @@ -157,6 +173,15 @@ public class OsgiImpl implements Osgi { return jdiscOsgi.isFelixFramework(); } + private static Bundle getSystemBundle(List<Bundle> bundles, OsgiFramework jdiscOsgi) { + for (Bundle b : bundles) { + if (b instanceof Framework) + return b; + } + if (jdiscOsgi.isFelixFramework()) throw new IllegalStateException("No system bundle in " + bundles); + return null; + } + private static Bundle firstNonFrameworkBundle(List<Bundle> bundles) { for (Bundle b : bundles) { if (! (b instanceof Framework)) diff --git a/container-disc/pom.xml b/container-disc/pom.xml index a64349855b1..f84e1868724 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -46,7 +46,13 @@ <artifactId>assertj-core</artifactId> <scope>test</scope> </dependency> - <dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-lib</artifactId> <version>${project.version}</version> @@ -105,15 +111,23 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>defaults</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespalog</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/FilterBindingsProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/FilterBindingsProvider.java index ac47bb9d03e..6e5ad367590 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/FilterBindingsProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/FilterBindingsProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java index 39c4b108aa6..280e10f7022 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java @@ -7,7 +7,7 @@ import ai.vespa.cloud.Environment; import ai.vespa.cloud.Node; import ai.vespa.cloud.SystemInfo; import ai.vespa.cloud.Zone; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ConfigserverConfig; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ZoneInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ZoneInfoProvider.java index 864d44886cb..c7dac91b73f 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ZoneInfoProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ZoneInfoProvider.java @@ -5,7 +5,7 @@ import ai.vespa.cloud.ApplicationId; import ai.vespa.cloud.Environment; import ai.vespa.cloud.Zone; import ai.vespa.cloud.ZoneInfo; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java index 696f76e8228..fff8a4dc48e 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.MetricConsumerFactory; import com.yahoo.jdisc.application.MetricConsumer; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java index 865fa621123..6d12a9251ee 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.jdisc.MetricConsumerFactory; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 4659c2acc36..f4e3abb44ed 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; diff --git a/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java b/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java index b845c0d0a7c..e74d1fec5a5 100644 --- a/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java +++ b/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.container.Container; import com.yahoo.container.jdisc.JdiscBindingsConfig; diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 19d54b2cfea..4ee98484438 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -27,8 +27,15 @@ else fi CONTAINER_HOME="${VESPA_HOME}/var/jdisc_container/${DISCRIMINATOR}/" -ZOOKEEPER_LOG_FILE_PREFIX="${VESPA_HOME}/logs/vespa/zookeeper.${VESPA_SERVICE_NAME}" -rm -f ZOOKEEPER_LOG_FILE_PREFIX*lck +if [[ "$VESPA_SERVICE_NAME" = "container" || "$VESPA_SERVICE_NAME" = "container-clustercontroller" || "$VESPA_SERVICE_NAME" = "qrserver" ]]; then + ZOOKEEPER_LOG_FILE_PREFIX="${VESPA_HOME}/logs/vespa/zookeeper.${VESPA_SERVICE_NAME}" + rm -f $ZOOKEEPER_LOG_FILE_PREFIX*lck + zookeeper_log_file_property="-Dzookeeper_log_file_prefix=${ZOOKEEPER_LOG_FILE_PREFIX}" +# TODO: Temporary, remove else clause after 2022-05-20 +else + ZOOKEEPER_LOG_FILE_PREFIX="${VESPA_HOME}/logs/vespa/zookeeper.${VESPA_SERVICE_NAME}" + rm -f $ZOOKEEPER_LOG_FILE_PREFIX* +fi # common setup export VESPA_LOG_TARGET=file:${VESPA_HOME}/logs/vespa/vespa.log @@ -295,12 +302,12 @@ exec $numactlcmd $envcmd java \ -Djdisc.export.packages=${jdisc_export_packages} \ -Djdisc.cache.path="$bundlecachedir" \ -Djdisc.bundle.path="${VESPA_HOME}/lib/jars" \ - -Djdisc.logger.enabled=false \ - -Djdisc.logger.level=ALL \ + -Djdisc.logger.enabled=true \ + -Djdisc.logger.level=WARNING \ -Djdisc.logger.tag="${VESPA_CONFIG_ID}" \ -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger \ -Dvespa.log.control.dir="${VESPA_LOG_CONTROL_DIR}" \ - -Dzookeeper_log_file_prefix="${ZOOKEEPER_LOG_FILE_PREFIX}" \ + ${zookeeper_log_file_property} \ -Dfile.encoding=UTF-8 \ -cp "$CP" \ "$@" \ diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java index af9edc31c5f..82f412d1d3a 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.messagebus; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.jdisc.ReferencedResource; diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java index fc7c82520a4..14f28ee4fae 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/NetworkMultiplexerProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.messagebus; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.messagebus.network.Identity; diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 3045247a6d3..b092f4fce10 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.messagebus; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.document.DocumentTypeManager; diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 7571cfb0969..ac4276d0325 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.jdisc; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.ResourceReference; diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 23ba3b0619c..ab29f98898a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.cluster; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.provider.ComponentRegistry; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index e96a83b39fe..a847a84e134 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.prelude.querytransform; import java.util.*; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.prelude.Index; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java index fe6087a3198..b67ddade7ee 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index ceb91a961d3..9bebaf98665 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 5dcd591b1fc..8ae1cbf5639 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -2,7 +2,7 @@ package com.yahoo.prelude.searcher; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java index e1cefae4b27..6c39bc299fb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.searcher; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.container.QrSearchersConfig; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index 8beb8b8462f..44c490b8ed1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -6,7 +6,7 @@ import java.util.Iterator; import java.util.List; import java.util.ListIterator; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index a8167fd2001..e25f00bdc80 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.language.Linguistics; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index e09b1d51733..ef50148d4d3 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; import com.yahoo.compress.Compressor; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index b026eadd843..09628db1284 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -2,7 +2,7 @@ package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 282eb8907a0..c0f4060be45 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -2,7 +2,7 @@ package com.yahoo.search.federation; import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.collections.Pair; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java index fcfae5de184..35cf72d6124 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java index 1dee75601b6..c23627accf4 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.pagetemplates; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.processing.IllegalInputException; diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/ResolverRegistry.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/ResolverRegistry.java index bd7c0bca5f3..e39cb8f9bc8 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/ResolverRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/ResolverRegistry.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.pagetemplates.engine.resolvers; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 19e0e441359..abd23c1822d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -147,7 +147,7 @@ public class QueryProfileProperties extends Properties { private String toShortString(Object value) { if ( ! (value instanceof Tensor)) return value.toString(); - return ((Tensor)value).toShortString(); + return ((Tensor)value).toAbbreviatedString(); } private Object convertByType(CompoundName name, Object value, Map<String, String> context) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index 39ec3e1c647..84246b1eb25 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.search.query.profile.QueryProfile; diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java index 76462559c87..6189202625a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.rewrite; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.search.*; import com.yahoo.config.*; import com.yahoo.search.query.rewrite.RewritesConfig.FsaDict; diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java index 0c2c08d1a1c..a7a723e47dc 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java @@ -5,7 +5,7 @@ import java.io.*; import java.util.*; import java.util.logging.Logger; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.fsa.FSA; import com.yahoo.search.query.rewrite.*; diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/MisspellRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/MisspellRewriter.java index 81c4dbc6919..ab26cad0312 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/MisspellRewriter.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/MisspellRewriter.java @@ -5,7 +5,7 @@ import java.io.*; import java.util.*; import java.util.logging.Logger; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.search.query.rewrite.*; diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/NameRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/NameRewriter.java index 820b1ab0576..543bcd092b7 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/NameRewriter.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/NameRewriter.java @@ -5,7 +5,7 @@ import java.io.*; import java.util.*; import java.util.logging.Logger; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.search.query.rewrite.*; import com.yahoo.search.*; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java index fff8935eefb..d3b166e03d0 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java @@ -14,7 +14,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import java.util.List; /** diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 5074a520a4e..aa561f58d7a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -26,7 +26,7 @@ public class Coverage extends com.yahoo.container.handler.Coverage { /** * Will set number of documents present in ideal state * - * @param soonActive Number of documents active in ideal state + * @param soonActive number of documents active in ideal state * @return self for chaining */ @Beta @@ -35,10 +35,11 @@ public class Coverage extends com.yahoo.container.handler.Coverage { /** * Will set the reasons for degraded coverage as reported by vespa backend. * - * @param degradedReason Reason for degradation + * @param degradedReason reason for degradation * @return self for chaining */ public Coverage setDegradedReason(int degradedReason) { this.degradedReason = degradedReason; return this; } public Coverage setNodesTried(int nodesTried) { super.setNodesTried(nodesTried); return this; } + } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index bf8ef39001f..69229c38818 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.searchchain; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.api.annotations.Beta; import com.yahoo.component.AbstractComponent; import com.yahoo.component.chain.Chain; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java index c39704bd6dc..dee00691fb3 100755 --- a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.searchers; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.metrics.simple.MetricReceiver; diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index 93cac27059e..66ad7b84608 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -2,7 +2,7 @@ package com.yahoo.search.yql; import com.yahoo.api.annotations.Beta; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.processing.IllegalInputException; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index be32b74591b..52f687e5708 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; @@ -14,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidator; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; -import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerRegistry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; @@ -36,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClien import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.time.Clock; +import java.util.Optional; /** * This provides access to all service dependencies of the controller. Implementations of this are responsible for @@ -99,7 +101,7 @@ public interface ServiceRegistry { BillingDatabaseClient billingDatabase(); - ContainerRegistry containerRegistry(); + Optional<? extends ArtifactRegistry> artifactRegistry(CloudName cloudName); TenantSecretService tenantSecretService(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/container/ContainerImage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/artifact/Artifact.java index 9652eeef530..7ca372f6cd0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/container/ContainerImage.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/artifact/Artifact.java @@ -1,33 +1,32 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.container; +package com.yahoo.vespa.hosted.controller.api.integration.artifact; import com.yahoo.component.Version; import java.time.Instant; import java.util.Objects; -import java.util.Optional; /** - * A container image. + * A registry artifact (e.g. container image or RPM) * * @author mpolden */ -public class ContainerImage { +public class Artifact { private final String id; private final String registry; private final String repository; + private final String tag; private final Instant createdAt; private final Version version; - private final Optional<Architecture> architecture; - public ContainerImage(String id, String registry, String repository, Instant createdAt, Version version, Optional<Architecture> architecture) { + public Artifact(String id, String registry, String repository, String tag, Instant createdAt, Version version) { this.id = Objects.requireNonNull(id); this.registry = Objects.requireNonNull(registry); this.repository = Objects.requireNonNull(repository); + this.tag = Objects.requireNonNull(tag); this.createdAt = Objects.requireNonNull(createdAt); this.version = Objects.requireNonNull(version); - this.architecture = Objects.requireNonNull(architecture); } /** Unique identifier of this */ @@ -35,16 +34,21 @@ public class ContainerImage { return id; } - /** The registry holding this image */ + /** The registry holding this artifact */ public String registry() { return registry; } - /** Repository of this image */ + /** Repository of this artifact */ public String repository() { return repository; } + /** Tag of this artifact */ + public String tag() { + return tag; + } + /** The time this was created */ public Instant createdAt() { return createdAt; @@ -55,43 +59,26 @@ public class ContainerImage { return version; } - /** The architecture of this, if any */ - public Optional<Architecture> architecture() { - return architecture; - } - - /** The tag of this image */ - public String tag() { - return version().toFullString() + architecture.map(arch -> "-" + arch.name()).orElse(""); - } - @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ContainerImage that = (ContainerImage) o; + Artifact that = (Artifact) o; return id.equals(that.id) && registry.equals(that.registry) && repository.equals(that.repository) && + tag.equals(that.tag) && createdAt.equals(that.createdAt) && - version.equals(that.version) && - architecture.equals(that.architecture); + version.equals(that.version); } @Override public int hashCode() { - return Objects.hash(id, registry, repository, createdAt, version, architecture); + return Objects.hash(id, registry, repository, tag, createdAt, version); } @Override public String toString() { - return "container image " + repository + " [registry=" + registry + ",version=" + version.toFullString() + - ",createdAt=" + createdAt + ",architecture=" + architecture.map(Enum::name).orElse("<none>") + "]"; + return "artifact " + registry + "/" + repository + " [version=" + version.toFullString() + ",createdAt=" + createdAt + ",tag=" + tag + "]"; } - - public enum Architecture { - amd64, - arm64, - } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/artifact/ArtifactRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/artifact/ArtifactRegistry.java new file mode 100644 index 00000000000..6ab8409ad11 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/artifact/ArtifactRegistry.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.artifact; + + +import java.util.List; + +/** + * A registry of artifacts (e.g. container image or RPM). + * + * @author mpolden + */ +public interface ArtifactRegistry { + + /** Delete all given artifacts */ + void deleteAll(List<Artifact> artifacts); + + /** Returns a list of all artifacts in this system */ + List<Artifact> list(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/container/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/artifact/package-info.java index ca1f6afc5db..8e12be06583 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/container/package-info.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/artifact/package-info.java @@ -3,6 +3,6 @@ * @author mpolden */ @ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.container; +package com.yahoo.vespa.hosted.controller.api.integration.artifact; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java index 19eaaa0da87..54fda58d19c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.athenz; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.ZmsClient; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index bb97349e5dd..a381bcd5bed 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -57,6 +57,7 @@ public class Node { private final String clusterId; private final ClusterType clusterType; private final String group; + private final int index; private final boolean retired; private final boolean wantToRetire; private final boolean wantToDeprovision; @@ -79,7 +80,7 @@ public class Node { Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, int cost, int failCount, Optional<String> flavor, String clusterId, - ClusterType clusterType, String group, boolean retired, boolean wantToRetire, boolean wantToDeprovision, + ClusterType clusterType, String group, int index, boolean retired, boolean wantToRetire, boolean wantToDeprovision, boolean wantToRebuild, boolean down, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, String> reports, List<Event> history, Set<String> ipAddresses, Set<String> additionalIpAddresses, @@ -111,6 +112,7 @@ public class Node { this.clusterType = Objects.requireNonNull(clusterType, "clusterType must be non-null"); this.retired = retired; this.group = Objects.requireNonNull(group, "group must be non-null"); + this.index = index; this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo must be non-null"); @@ -263,9 +265,10 @@ public class Node { } /** The group of this node, empty string if unallocated */ - public String group() { - return group; - } + public String group() { return group; } + + /** The membership index of this node */ + public int index() { return index; } /** Whether this node has been requested to retire */ public boolean wantToRetire() { @@ -283,9 +286,7 @@ public class Node { } /** Whether this node is currently down */ - public boolean down() { - return down; - } + public boolean down() { return down; } /** The tenant this has been reserved to, if any */ public Optional<TenantName> reservedTo() { return reservedTo; } @@ -376,7 +377,7 @@ public class Node { combined, unknown } - + /** Known nope environments */ public enum Environment { bareMetal, @@ -470,6 +471,7 @@ public class Node { private String clusterId = ""; private ClusterType clusterType = ClusterType.unknown; private String group = ""; + private int index = 0; private boolean retired = false; private boolean wantToRetire = false; private boolean wantToDeprovision = false; @@ -516,6 +518,7 @@ public class Node { this.clusterId = node.clusterId; this.clusterType = node.clusterType; this.group = node.group; + this.index = node.index; this.retired = node.retired; this.wantToRetire = node.wantToRetire; this.wantToDeprovision = node.wantToDeprovision; @@ -676,6 +679,11 @@ public class Node { return this; } + public Builder index(int index) { + this.index = index; + return this; + } + public Builder retired(boolean retired) { this.retired = retired; return this; @@ -755,7 +763,7 @@ public class Node { return new Node(id, hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, - wantedRebootGeneration, cost, failCount, flavor, clusterId, clusterType, group, retired, + wantedRebootGeneration, cost, failCount, flavor, clusterId, clusterType, group, index, retired, wantToRetire, wantToDeprovision, wantToRebuild, down, reservedTo, exclusiveTo, wantedDockerImage, currentDockerImage, reports, history, ipAddresses, additionalIpAddresses, additionalHostnames, switchHostname, modelName, environment); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/container/ContainerRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/container/ContainerRegistry.java deleted file mode 100644 index 78757ad995a..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/container/ContainerRegistry.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.container; - - -import java.util.List; - -/** - * A registry of container images. - * - * @author mpolden - */ -public interface ContainerRegistry { - - /** Delete all given images */ - void deleteAll(List<ContainerImage> images); - - /** Returns a list of all container images in this system */ - List<ContainerImage> list(); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java index a62f43d1cf5..b0fed17bdea 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueInfo.Status; import java.io.InputStream; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java index bfe833740d1..20178f300d2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 7ac7a36d742..8b4c00e9b9d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -37,6 +37,9 @@ public interface ZoneRegistry { /** Returns a list containing the id of all zones in this registry */ ZoneFilter zones(); + /** Returns a list containing the id of all zones in this registry, including the system. */ + ZoneFilter zonesIncludingSystem(); + /** Returns the default region for the given environment, if one is configured */ Optional<RegionName> getDefaultRegion(Environment environment); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index eb8f6ed6ae2..1c4f0994f8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.component.Vtag; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 6b93229dffe..c88eb2f1b86 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.impl; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.client.ErrorHandler; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 2ad28893e18..e4869cc9bf4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 1d56e2db08b..30f16acf77d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; @@ -40,9 +39,10 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; +import java.util.ArrayDeque; import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -55,7 +55,6 @@ import java.util.TreeMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.logging.Logger; @@ -390,12 +389,13 @@ public class JobController { * Throws TimeoutException if some step in this job is still being run. */ public void finish(RunId id) throws TimeoutException { - List<Mutex> locks = new ArrayList<>(); + Deque<Mutex> locks = new ArrayDeque<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. Run unlockedRun = run(id).get(); + locks.push(curator.lock(id.application(), id.type(), report)); for (Step step : report.allPrerequisites(unlockedRun.steps().keySet())) - locks.add(curator.lock(id.application(), id.type(), step)); + locks.push(curator.lock(id.application(), id.type(), step)); locked(id, run -> { // If run should be reset, just return here. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 82d154dcf03..379cc9c4f0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.List; import java.util.stream.Stream; +import static java.util.Comparator.reverseOrder; import static java.util.stream.Collectors.toList; /** @@ -87,7 +88,7 @@ public enum Step { .filter(among::contains) .flatMap(pre -> Stream.concat(Stream.of(pre), pre.allPrerequisites(among).stream())) - .sorted() + .sorted(reverseOrder()) .distinct() .collect(toList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index e3f69f59d24..bd69ea41b05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -46,13 +46,12 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { @Override protected double maintain() { // Count buckets - so we can alert if we get close to the account limit of 1000 - zoneRegistry.zones().all().ids().forEach(zoneId -> + zoneRegistry.zonesIncludingSystem().all().ids().forEach(zoneId -> metric.set(bucketCountMetricName, archiveBucketDb.buckets(zoneId).size(), metric.createContext(Map.of("zone", zoneId.value())))); - - zoneRegistry.zones().controllerUpgraded().zones().forEach(z -> { - ZoneId zoneId = z.getId(); + zoneRegistry.zonesIncludingSystem().controllerUpgraded().zones().forEach(z -> { + ZoneId zoneId = z.getVirtualId(); try { var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java new file mode 100644 index 00000000000..81fbb85e3d7 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact; +import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; + +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Periodically expire unused artifacts, e.g. container images and RPMs. + * + * @author mpolden + */ +public class ArtifactExpirer extends ControllerMaintainer { + + private static final Logger log = Logger.getLogger(ArtifactExpirer.class.getName()); + + private static final Duration MIN_AGE = Duration.ofDays(14); + + public ArtifactExpirer(Controller controller, Duration interval) { + super(controller, interval, null, expiringSystems()); + } + + @Override + protected double maintain() { + VersionStatus versionStatus = controller().readVersionStatus(); + return controller().clouds().stream() + .flatMapToDouble(cloud -> + controller().serviceRegistry().artifactRegistry(cloud).stream() + .mapToDouble(artifactRegistry -> maintain(versionStatus, cloud, artifactRegistry))) + .average() + .orElse(1); + } + + private double maintain(VersionStatus versionStatus, CloudName cloudName, ArtifactRegistry artifactRegistry) { + try { + Instant now = controller().clock().instant(); + List<Artifact> artifactsToExpire = artifactRegistry.list().stream() + .filter(artifact -> isExpired(artifact, now, versionStatus)) + .collect(Collectors.toList()); + if (!artifactsToExpire.isEmpty()) { + log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts: " + artifactsToExpire); + artifactRegistry.deleteAll(artifactsToExpire); + } + return 1; + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e); + return 0; + } + } + + /** Returns whether given artifact is expired */ + private boolean isExpired(Artifact artifact, Instant now, VersionStatus versionStatus) { + List<VespaVersion> versions = versionStatus.versions(); + if (versions.isEmpty()) return false; + + if (versionStatus.isActive(artifact.version())) return false; + if (artifact.createdAt().isAfter(now.minus(MIN_AGE))) return false; + + Version maxVersion = versions.stream().map(VespaVersion::versionNumber).max(Comparator.naturalOrder()).get(); + if (artifact.version().isAfter(maxVersion)) return false; // A future version + + return true; + } + + /** Returns systems where artifacts can be expired */ + private static Set<SystemName> expiringSystems() { + // Run only in public and main. Public systems have distinct container registries, while main and CD have + // shared registries. + return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java deleted file mode 100644 index c87dd262fa3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Periodically expire unused container images. - * - * @author mpolden - */ -public class ContainerImageExpirer extends ControllerMaintainer { - - private static final Logger log = Logger.getLogger(ContainerImageExpirer.class.getName()); - - private static final Duration MIN_AGE = Duration.ofDays(14); - - public ContainerImageExpirer(Controller controller, Duration interval) { - super(controller, interval, null, expiringSystems()); - } - - @Override - protected double maintain() { - Instant now = controller().clock().instant(); - VersionStatus versionStatus = controller().readVersionStatus(); - List<ContainerImage> imagesToExpire = controller().serviceRegistry().containerRegistry().list().stream() - .filter(image -> isExpired(image, now, versionStatus)) - .collect(Collectors.toList()); - if (!imagesToExpire.isEmpty()) { - log.log(Level.INFO, "Expiring " + imagesToExpire.size() + " container images: " + imagesToExpire); - controller().serviceRegistry().containerRegistry().deleteAll(imagesToExpire); - } - return 1.0; - } - - /** Returns whether given image is expired */ - private boolean isExpired(ContainerImage image, Instant now, VersionStatus versionStatus) { - List<VespaVersion> versions = versionStatus.versions(); - if (versions.isEmpty()) return false; - - if (versionStatus.isActive(image.version())) return false; - if (image.createdAt().isAfter(now.minus(MIN_AGE))) return false; - - Version maxVersion = versions.stream().map(VespaVersion::versionNumber).max(Comparator.naturalOrder()).get(); - if (image.version().isAfter(maxVersion)) return false; // A future version - - return true; - } - - /** Returns systems where images can be expired */ - private static Set<SystemName> expiringSystems() { - // Run only in public and main. Public systems have distinct container registries, while main and CD have - // shared registries. - return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 041d0694ca9..193d171e334 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.config.provision.SystemName; @@ -63,7 +63,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger())); maintainers.add(new SystemRoutingPolicyMaintainer(controller, intervals.systemRoutingPolicyMaintainer)); maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector)); - maintainers.add(new ContainerImageExpirer(controller, intervals.containerImageExpirer)); + maintainers.add(new ArtifactExpirer(controller, intervals.containerImageExpirer)); maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater)); maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 193fb89eb99..f3256237284 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Sets; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 1463cce595d..551f803f368 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -214,6 +214,10 @@ public class VcmrMaintainer extends ControllerMaintainer { return hostAction.withState(State.PENDING_RETIREMENT); } + if (isFailed(node)) { + return hostAction.withState(State.NONE); + } + return hostAction; } @@ -260,6 +264,11 @@ public class VcmrMaintainer extends ControllerMaintainer { action.getState() == State.RETIRING && !node.wantToRetire(); } + private boolean isFailed(Node node) { + return node.state() == Node.State.failed || + node.state() == Node.State.breakfixed; + } + private Map<ZoneId, List<Node>> nodesByZone() { return controller().zoneRegistry() .zones() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index c3c68f7596f..f02f49e7114 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.concurrent.UncheckedTimeoutException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index 21414339a87..8f36daf9756 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.curator.mock.MockCurator; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index dffc7bf43b4..671222e2123 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.text.Text; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 41055339ac6..e50f81919df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -10,7 +10,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; @@ -29,6 +29,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.restapi.ByteArrayResponse; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; @@ -72,7 +73,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; -import com.yahoo.jdisc.http.filter.security.misc.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -1120,6 +1120,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); nodeObject.setString("group", node.group()); + nodeObject.setLong("index", node.index()); } return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java index 9e218ce89e6..56d82d286cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.athenz; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ResourceResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 3f610d27950..206303adc80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -29,9 +29,6 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; -import javax.ws.rs.BadRequestException; -import javax.ws.rs.ForbiddenException; -import javax.ws.rs.NotFoundException; import java.io.IOException; import java.math.BigDecimal; import java.security.Principal; @@ -71,23 +68,25 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { + Optional<String> userId = Optional.ofNullable(request.getJDiscRequest().getUserPrincipal()).map(Principal::getName); + if (userId.isEmpty()) + return ErrorResponse.unauthorized("Must be authenticated to use this API"); + Path path = new Path(request.getUri()); - String userId = userIdOrThrow(request); switch (request.getMethod()) { case GET: - return handleGET(request, path, userId); + return handleGET(request, path, userId.get()); case PATCH: - return handlePATCH(request, path, userId); + return handlePATCH(request, path, userId.get()); case DELETE: - return handleDELETE(path, userId); + return handleDELETE(path, userId.get()); case POST: - return handlePOST(path, request, userId); + return handlePOST(path, request, userId.get()); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } - } catch (NotFoundException e) { - return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); - } catch (IllegalArgumentException e) { + } + catch (IllegalArgumentException e) { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (Exception e) { log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); @@ -142,8 +141,8 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { private HttpResponse handlePOST(Path path, HttpRequest request, String userId) { if (path.matches("/billing/v1/invoice")) return createBill(request, userId); - if (path.matches("/billing/v1/invoice/{invoice-id}/status")) return setBillStatus(request, path.get("invoice-id")); - if (path.matches("/billing/v1/invoice/tenant/{tenant}/line-item")) return addLineItem(request, path.get("tenant")); + if (path.matches("/billing/v1/invoice/{invoice-id}/status")) return setBillStatus(request, path.get("invoice-id"), userId); + if (path.matches("/billing/v1/invoice/tenant/{tenant}/line-item")) return addLineItem(request, path.get("tenant"), userId); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -227,20 +226,20 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { tc.setString("collection", collection.name()); } - private HttpResponse addLineItem(HttpRequest request, String tenant) { + private HttpResponse addLineItem(HttpRequest request, String tenant, String userId) { Inspector inspector = inspectorOrThrow(request); billingController.addLineItem( TenantName.from(tenant), getInspectorFieldOrThrow(inspector, "description"), new BigDecimal(getInspectorFieldOrThrow(inspector, "amount")), - userIdOrThrow(request)); + userId); return new MessageResponse("Added line item for tenant " + tenant); } - private HttpResponse setBillStatus(HttpRequest request, String billId) { + private HttpResponse setBillStatus(HttpRequest request, String billId, String userId) { Inspector inspector = inspectorOrThrow(request); String status = getInspectorFieldOrThrow(inspector, "status"); - billingController.updateBillStatus(Bill.Id.of(billId), userIdOrThrow(request), status); + billingController.updateBillStatus(Bill.Id.of(billId), userId, status); return new MessageResponse("Updated status of invoice " + billId); } @@ -456,19 +455,13 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { try { return SlimeUtils.jsonToSlime(request.getData().readAllBytes()).get(); } catch (IOException e) { - throw new BadRequestException("Failed to parse request body"); + throw new IllegalArgumentException("Failed to parse request body"); } } - private static String userIdOrThrow(HttpRequest request) { - return Optional.ofNullable(request.getJDiscRequest().getUserPrincipal()) - .map(Principal::getName) - .orElseThrow(() -> new ForbiddenException("Must be authenticated to use this API")); - } - private static String getInspectorFieldOrThrow(Inspector inspector, String field) { if (!inspector.field(field).valid()) - throw new BadRequestException("Field " + field + " cannot be null"); + throw new IllegalArgumentException("Field " + field + " cannot be null"); return inspector.field(field).asString(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 087d6152afc..419d1f4cb33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -25,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import javax.ws.rs.BadRequestException; import java.math.BigDecimal; import java.time.Clock; import java.time.LocalDate; @@ -76,6 +76,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) + .addExceptionMapper(RuntimeException.class, (__, e) -> ErrorResponse.internalServerError(e.getMessage())) .build(); } @@ -329,7 +330,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler private static String getInspectorFieldOrThrow(Inspector inspector, String field) { if (!inspector.field(field).valid()) - throw new BadRequestException("Field " + field + " cannot be null"); + throw new RestApiException.BadRequest("Field " + field + " cannot be null"); return inspector.field(field).asString(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 01bd02fdc13..4aefb9ea7c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.auth0.jwt.JWT; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 7c695ef51d7..cef6840dfe1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java index 4762d2c4612..e840b70a95a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index b7c77e7bfb4..5eaa6d7af1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import ai.vespa.hosted.api.Method; import ai.vespa.hosted.api.RequestVerifier; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java index 708891c8251..f9f3025837d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/horizon/HorizonApiHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.horizon; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index e0097c295c7..aaaf09fa781 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.systemflags; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 025e8dff659..fce2d283da2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.user; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java index f4f6df28ebc..4ece2b9a691 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AthenzAccessControlRequests.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.Inspector; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 39ee2a6ce44..87691d2927a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.security; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index a08ec77bbb9..9bf0f8dcde2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.security.KeyStoreBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRegistryMock.java new file mode 100644 index 00000000000..ef0445e06c6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRegistryMock.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.integration; + +import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact; +import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author mpolden + */ +public class ArtifactRegistryMock implements ArtifactRegistry { + + private static final Comparator<Artifact> comparator = Comparator.comparing(Artifact::registry) + .thenComparing(Artifact::repository) + .thenComparing(Artifact::version); + + private final Map<String, Artifact> images = new HashMap<>(); + + public ArtifactRegistryMock add(Artifact image) { + images.put(image.id(), image); + return this; + } + + @Override + public void deleteAll(List<Artifact> artifacts) { + artifacts.forEach(image -> this.images.remove(image.id())); + } + + @Override + public List<Artifact> list() { + return images.values().stream().sorted(comparator).collect(Collectors.toUnmodifiableList()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 1ed84659f58..b3689dacea7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.integration; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL.Path; import ai.vespa.http.HttpURL.Query; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java deleted file mode 100644 index 4791017548d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.integration; - -import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage; -import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerRegistry; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * @author mpolden - */ -public class ContainerRegistryMock implements ContainerRegistry { - - private static final Comparator<ContainerImage> comparator = Comparator.comparing(ContainerImage::registry) - .thenComparing(ContainerImage::repository) - .thenComparing(ContainerImage::version); - - private final Map<String, ContainerImage> images = new HashMap<>(); - - public ContainerRegistryMock add(ContainerImage image) { - images.put(image.id(), image); - return this; - } - - @Override - public void deleteAll(List<ContainerImage> images) { - images.forEach(image -> this.images.remove(image.id())); - } - - @Override - public List<ContainerImage> list() { - return images.values().stream().sorted(comparator).collect(Collectors.toUnmodifiableList()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 0afb1e67a85..a91d81bb0c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.test.ManualClock; @@ -50,6 +51,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainerMock import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; import java.time.Instant; +import java.util.Optional; /** * A mock implementation of a {@link ServiceRegistry} for testing purposes. @@ -82,7 +84,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); private final RoleService roleService = new MockRoleService(); private final BillingController billingController = new MockBillingController(clock); - private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock(); + private final ArtifactRegistryMock containerRegistry = new ArtifactRegistryMock(); private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); private final ArchiveService archiveService = new MockArchiveService(); private final MockChangeRequestClient changeRequestClient = new MockChangeRequestClient(); @@ -236,8 +238,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public ContainerRegistryMock containerRegistry() { - return containerRegistry; + public Optional<ArtifactRegistryMock> artifactRegistry(CloudName cloudName) { + return Optional.of(containerRegistry); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java index 9e08e774744..d173fcb0e18 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -35,15 +35,15 @@ public class ZoneApiMock implements ZoneApi { } public static ZoneApiMock fromId(String id) { - return newBuilder().withId(id).build(); + return from(ZoneId.from(id)); } public static ZoneApiMock from(Environment environment, RegionName region) { - return newBuilder().with(ZoneId.from(environment, region)).build(); + return from(ZoneId.from(environment, region)); } - public static ZoneApiMock from(ZoneId zone) { - return newBuilder().with(zone).build(); + public static ZoneApiMock from(ZoneId id) { + return newBuilder().with(id).build(); } @Override @@ -80,7 +80,7 @@ public class ZoneApiMock implements ZoneApi { private SystemName systemName = SystemName.defaultSystem(); private ZoneId id = ZoneId.defaultId(); - private ZoneId virtualId ; + private ZoneId virtualId = null; private CloudName cloudName = CloudName.defaultName(); private String cloudNativeRegionName = id.region().value(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 5b8e25cbfe8..849503ae8d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.time.Duration; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -51,7 +52,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry /** * This sets the default list of zones contained in this. If your test need a particular set of zones, use - * {@link #setZones(List)} instead of changing the default set.} + * {@link #setZones(List)} instead of changing the default set. */ public ZoneRegistryMock(SystemName system) { this.system = system; @@ -156,6 +157,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public ZoneFilter zonesIncludingSystem() { + var fullZones = new ArrayList<ZoneApi>(1 + zones.size()); + fullZones.add(systemAsZone()); + fullZones.addAll(zones); + return ZoneFilterMock.from(fullZones, zoneRoutingMethods, reprovisionToUpgradeOs); + } + + private ZoneApiMock systemAsZone() { + return ZoneApiMock.newBuilder() + .with(ZoneId.from("prod.us-east-1")) + .withVirtualId(ZoneId.from("prod.controller")) + .build(); + } + + @Override public AthenzService getConfigServerHttpsIdentity(ZoneId zone) { return new AthenzService("vespadomain", "provider-" + zone.environment().value() + "-" + zone.region().value()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java index df2b462914e..5571f957e83 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -48,10 +48,10 @@ public class ArchiveAccessMaintainerTest { assertEquals(Set.of(tenant1role), archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); var expected = Map.of("archive.bucketCount", - tester.controller().zoneRegistry().zones().all().ids().stream() - .collect(Collectors.toMap( - zone -> Map.of("zone", zone.value()), - zone -> zone.equals(testZone) ? 1d : 0d))); + tester.controller().zoneRegistry().zonesIncludingSystem().all().ids().stream() + .collect(Collectors.toMap( + zone -> Map.of("zone", zone.value()), + zone -> zone.equals(testZone) ? 1d : 0d))); assertEquals(expected, metric.metrics()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirerTest.java index baebf7239ee..7703266c9ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirerTest.java @@ -2,35 +2,33 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage; -import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage.Architecture; +import com.yahoo.config.provision.CloudName; +import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.integration.ContainerRegistryMock; +import com.yahoo.vespa.hosted.controller.integration.ArtifactRegistryMock; import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertEquals; /** * @author mpolden */ -public class ContainerImageExpirerTest { +public class ArtifactExpirerTest { @Test public void maintain() { DeploymentTester tester = new DeploymentTester(); - ContainerImageExpirer expirer = new ContainerImageExpirer(tester.controller(), Duration.ofDays(1)); - ContainerRegistryMock registry = tester.controllerTester().serviceRegistry().containerRegistry(); + ArtifactExpirer expirer = new ArtifactExpirer(tester.controller(), Duration.ofDays(1)); + ArtifactRegistryMock registry = tester.controllerTester().serviceRegistry().artifactRegistry(CloudName.defaultName()).orElseThrow(); Instant instant = tester.clock().instant(); - ContainerImage image0 = new ContainerImage("image0", "registry.example.com", "vespa/vespa", instant, Version.fromString("7.1"), Optional.empty()); - ContainerImage image1 = new ContainerImage("image1", "registry.example.com", "vespa/vespa", instant, Version.fromString("7.2"), Optional.of(Architecture.amd64)); - ContainerImage image2 = new ContainerImage("image2", "registry.example.com", "vespa/vespa", instant, Version.fromString("7.4"), Optional.of(Architecture.amd64)); - + Artifact image0 = new Artifact("image0", "registry.example.com", "vespa/vespa", "7.1", instant, Version.fromString("7.1")); + Artifact image1 = new Artifact("image1", "registry.example.com", "vespa/vespa", "7.2-amd64", instant, Version.fromString("7.2")); + Artifact image2 = new Artifact("image2", "registry.example.com", "vespa/vespa", "7.4-amd64", instant, Version.fromString("7.4")); registry.add(image0) .add(image1) .add(image2); @@ -53,7 +51,7 @@ public class ContainerImageExpirerTest { assertEquals(List.of(image1, image2), registry.list()); // A new version becomes active. The active and future version are kept - ContainerImage image3 = new ContainerImage("image3", "registry.example.com", "vespa/vespa", tester.clock().instant(), Version.fromString("7.3"), Optional.of(Architecture.amd64)); + Artifact image3 = new Artifact("image3", "registry.example.com", "vespa/vespa", "7.3-arm64", tester.clock().instant(), Version.fromString("7.3")); registry.add(image3); tester.controllerTester().upgradeSystem(image3.version()); expirer.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json index 88a92bf7810..34d2f054e0f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -18,7 +18,8 @@ "retired": false, "restarting": false, "rebooting": false, - "group": "" + "group": "", + "index": 0 } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 79e11fa1140..8b2e5578ae0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -13,6 +13,9 @@ "name": "ArchiveUriUpdater" }, { + "name": "ArtifactExpirer" + }, + { "name": "BillingDatabaseMaintainer" }, { @@ -28,9 +31,6 @@ "name": "ContactInformationMaintainer" }, { - "name": "ContainerImageExpirer" - }, - { "name": "CostReportMaintainer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java index 8b5cdefddf8..0de065b3eaf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecretStoreMock.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tls; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.hosted.controller.tls.config.TlsConfig; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index bff0e881c3d..12f71c4fc84 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -396,6 +396,13 @@ public class Flags { "Takes effect immediately", APPLICATION_ID); + public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( + "enable-proxy-protocol-mixed-mode", true, + List.of("tokle"), "2022-05-09", "2022-07-01", + "Enable or disable proxy protocol mixed mode", + "Takes effect on redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fnet/src/vespa/fnet/connector.cpp b/fnet/src/vespa/fnet/connector.cpp index 1d2002d6861..f5ce49f2019 100644 --- a/fnet/src/vespa/fnet/connector.cpp +++ b/fnet/src/vespa/fnet/connector.cpp @@ -20,14 +20,15 @@ FNET_Connector::FNET_Connector(FNET_TransportThread *owner, : FNET_IOComponent(owner, server_socket.get_fd(), spec, /* time-out = */ false), _streamer(streamer), _serverAdapter(serverAdapter), - _server_socket(std::move(server_socket)) + _server_socket(std::move(server_socket)), + _cached_port(_server_socket.address().port()) { } uint32_t FNET_Connector::GetPortNumber() const { - return _server_socket.address().port(); + return _cached_port; } diff --git a/fnet/src/vespa/fnet/connector.h b/fnet/src/vespa/fnet/connector.h index ff93a0a6a92..858313d6582 100644 --- a/fnet/src/vespa/fnet/connector.h +++ b/fnet/src/vespa/fnet/connector.h @@ -18,6 +18,7 @@ private: FNET_IPacketStreamer *_streamer; FNET_IServerAdapter *_serverAdapter; vespalib::ServerSocket _server_socket; + uint32_t _cached_port; FNET_Connector(const FNET_Connector &); FNET_Connector &operator=(const FNET_Connector &); diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java index a8c62cac872..b2713cfcee3 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java @@ -10,7 +10,7 @@ import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClient; import com.amazonaws.services.simplesystemsmanagement.model.GetParametersRequest; import com.amazonaws.services.simplesystemsmanagement.model.GetParametersResult; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java index 299bf2facc5..2318481f247 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStoreValidationHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.cloud.aws; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java index 4eac5c95141..5b774648314 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.athenz; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java index a302f19db99..97316f1d04a 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.athenz; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/StaticRequestResourceMapper.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/StaticRequestResourceMapper.java index 78aadbbd52f..627cb793569 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/StaticRequestResourceMapper.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/StaticRequestResourceMapper.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.athenz; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.vespa.athenz.api.AthenzResourceName; import java.util.Optional; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilter.java index d704fb926a6..e2efd2d220c 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.cors; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilter.java index 0ed239e7834..f56965ea6a8 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.cors; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.http.filter.DiscFilterResponse; import com.yahoo.jdisc.http.filter.RequestView; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java index c5ec08e23cb..dac4d3ee4d6 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.rule; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 928258b5e75..1943e2905fa 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -66,7 +66,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <version>${project.version}</version> - <scope>test</scope> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -171,7 +170,11 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> - <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> </dependency> </dependencies> <build> @@ -249,7 +252,11 @@ <argument>${project.build.directory}/dependency/slf4j-jdk14.jar</argument> <argument>${project.build.directory}/dependency/jcl-over-slf4j.jar</argument> <argument>${project.build.directory}/dependency/log4j-over-slf4j.jar</argument> + <argument>${project.build.directory}/dependency/annotations.jar</argument> <argument>${project.build.directory}/dependency/config-lib.jar</argument> + <argument>${project.build.directory}/dependency/defaults.jar</argument> + <argument>${project.build.directory}/dependency/vespajlib.jar</argument> + <argument>${project.build.directory}/dependency/vespalog.jar</argument> <argument>${project.build.directory}/dependency/yolean.jar</argument> <argument>${project.build.directory}/dependency/jaxb-api.jar</argument> <argument>${project.build.directory}/dependency/jaxb-core.jar</argument> @@ -279,6 +286,19 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>3.0.0-M6</version> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> <configuration> <archive> diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java new file mode 100644 index 00000000000..395ddc889ae --- /dev/null +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java @@ -0,0 +1,42 @@ +package com.yahoo.jdisc.core; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileReader; +import java.util.Properties; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Integration tests for {@link ExportPackages}. + * + * @author gjoranv + */ +public class ExportPackagesIT { + + @Rule + public TemporaryFolder tempFolder= new TemporaryFolder(); + + @Test + public void export_packages_are_added_from_dependency_jars() throws Exception { + File file = tempFolder.newFile(ExportPackages.PROPERTIES_FILE); + + ExportPackages.main(new String[] { file.getAbsolutePath(), "target/dependency/guice-no_aop.jar" }); + assertTrue(file.exists()); + Properties props = new Properties(); + String exportPackages; + try (FileReader reader = new FileReader(file)) { + props.load(reader); + exportPackages = props.getProperty(ExportPackages.EXPORT_PACKAGES); + } + assertNotNull(exportPackages); + + assertTrue(exportPackages.contains("com.google.inject")); + + } + +} diff --git a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java index 31964eac514..e3a0457d77a 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/sentencepiece/SentencePieceEmbedder.java @@ -2,7 +2,7 @@ package com.yahoo.language.sentencepiece; import com.yahoo.api.annotations.Beta; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.language.tools.Embed; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; diff --git a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java index 08de05f351a..55a0411cd8a 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java +++ b/linguistics-components/src/main/java/com/yahoo/language/wordpiece/WordPieceEmbedder.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.wordpiece; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.language.tools.Embed; import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index c749679024a..fd995c3fb78 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.opennlp; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index b10beb8c9af..42172be680b 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; diff --git a/logserver/pom.xml b/logserver/pom.xml index 409f3e51217..2d5eecd01ce 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -17,6 +17,11 @@ <!-- compile scope --> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>defaults</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> </dependency> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index 73396af5156..30acf6eacc8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -8,7 +8,7 @@ import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.handler.metrics.ErrorResponse; import com.yahoo.container.handler.metrics.HttpHandlerBase; import com.yahoo.container.handler.metrics.JsonResponse; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index 464379e7980..8d33ee19c78 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -4,7 +4,7 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import org.apache.hc.client5.http.ConnectTimeoutException; import org.apache.hc.client5.http.config.RequestConfig; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java index 86704038869..96fda02c155 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java @@ -6,7 +6,7 @@ import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.handler.metrics.ErrorResponse; import com.yahoo.container.handler.metrics.HttpHandlerBase; import com.yahoo.container.handler.metrics.JsonResponse; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java index 317f790b08e..468a61baa88 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java @@ -11,7 +11,7 @@ import ai.vespa.metricsproxy.http.application.ServiceIdDimensionProcessor; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.handler.metrics.ErrorResponse; import com.yahoo.container.handler.metrics.HttpHandlerBase; import com.yahoo.container.handler.metrics.JsonResponse; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java index 1b69053ebe8..54cb0da708c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java @@ -7,7 +7,7 @@ import ai.vespa.metricsproxy.http.TextResponse; import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.handler.metrics.HttpHandlerBase; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java index 10b3cc3077b..41c7542b613 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java @@ -11,7 +11,7 @@ import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; import ai.vespa.metricsproxy.node.NodeMetricGatherer; import ai.vespa.metricsproxy.service.VespaServices; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.handler.metrics.ErrorResponse; import com.yahoo.container.handler.metrics.HttpHandlerBase; import com.yahoo.container.handler.metrics.JsonResponse; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java index 9686bb1bf6b..e18ea374fe1 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java @@ -10,7 +10,7 @@ import ai.vespa.metricsproxy.metric.model.ServiceId; import ai.vespa.metricsproxy.service.SystemPollerProvider; import ai.vespa.metricsproxy.service.VespaServices; import com.fasterxml.jackson.databind.JsonNode; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.state.CoredumpGatherer; import com.yahoo.container.jdisc.state.FileWrapper; import com.yahoo.container.jdisc.state.HostLifeGatherer; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index 824c493e140..6d41fc93c25 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import java.util.logging.Level; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java index 115c279e6c8..4318a689ba8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java @@ -5,7 +5,7 @@ import ai.vespa.metricsproxy.core.MonitoringConfig; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.service.VespaServicesConfig.Service; import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import java.util.ArrayList; import java.util.Collections; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java index 8eb774527e6..aa35832a3be 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.telegraf; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import java.io.File; diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index 1554f11195f..88843fd99ab 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -3,7 +3,7 @@ package ai.vespa.models.evaluation; import com.yahoo.api.annotations.Beta; import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.vespa.config.search.RankProfilesConfig; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java index 5f641ce3996..f7fdf171fc1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.container.metrics; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import java.util.ArrayList; import java.util.HashMap; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 2271991fa15..0d0b8bd335e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; @@ -84,9 +85,17 @@ public class CoredumpHandler { public void converge(NodeAgentContext context, Supplier<Map<String, Object>> nodeAttributesSupplier, boolean throwIfCoreBeingWritten) { - ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer); + ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer, context.users().vespa()); ContainerPath containerProcessingPath = containerCrashPath.resolve(PROCESSING_DIRECTORY_NAME); + // TODO (freva): Remove after 7.584 + UnixUser vespaUser = context.users().vespa(); + UnixPath processingPath = new UnixPath(containerProcessingPath); + processingPath.getAttributesIfExists().ifPresent(attr -> { + if (attr.ownerId() != vespaUser.uid()) processingPath.setOwnerId(vespaUser.uid()); + if (attr.groupId() != vespaUser.gid()) processingPath.setGroupId(vespaUser.gid()); + }); + updateMetrics(context, containerCrashPath); if (throwIfCoreBeingWritten) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index a9f2cd219b9..ffce5e6e68b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -2,10 +2,13 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.sync; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import java.net.URI; import java.nio.file.Path; import java.time.Duration; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -15,6 +18,9 @@ import java.util.Optional; */ public class SyncFileInfo { + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter + .ofPattern("yyyy-MM-dd.HH-mm-ss").withZone(ZoneOffset.UTC); + private final Path source; private final URI destination; private final Compression uploadCompression; @@ -53,12 +59,19 @@ public class SyncFileInfo { String filename = logFile.getFileName().toString(); Compression compression; String dir = null; + String remoteFilename = logFile.getFileName().toString(); Duration minDurationBetweenSync = null; if (filename.startsWith("vespa.log")) { dir = "logs/vespa/"; compression = Compression.ZSTD; minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; + } else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) { + compression = Compression.ZSTD; + dir = "logs/zookeeper/"; + remoteFilename = filename.endsWith(".0.log") ? "zookeeper.log" : + "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); + minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; } else { compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD; if (rotatedOnly && compression != Compression.NONE) @@ -71,7 +84,7 @@ public class SyncFileInfo { if (dir == null) return Optional.empty(); return Optional.of(new SyncFileInfo( - logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression, defaultTags(owner), + logFile, uri.resolve(dir + remoteFilename + compression.extension), compression, defaultTags(owner), minDurationBetweenSync)); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index dff72fe81f1..a5edd3bd74d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -110,6 +110,7 @@ public class NodeAdminImpl implements NodeAdmin { if (!isSuspended) { Runtime runtime = Runtime.getRuntime(); + runtime.gc(); long freeMemory = runtime.freeMemory(); long totalMemory = runtime.totalMemory(); long usedMemory = totalMemory - freeMemory; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 61e777a9576..dbab7270f08 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -466,6 +466,7 @@ public class NodeAgentImpl implements NodeAgent { case failed: case inactive: case parked: + storageMaintainer.syncLogs(context, true); removeContainerIfNeededUpdateContainerState(context, container); updateNodeRepoWithCurrentAttributes(context, Optional.empty()); stopServicesIfNeeded(context); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index 54701b0f5a7..1b5c51082c5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.sync; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; @@ -9,6 +10,7 @@ import java.net.URI; import java.nio.file.FileSystem; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.SyncFileInfo.Compression.NONE; @@ -31,6 +33,8 @@ public class SyncFileInfoTest { private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/ConnectionLog.default.20210212.zst"); private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log"); private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); + private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log"); + private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); @Test public void access_logs() { @@ -65,6 +69,16 @@ public class SyncFileInfoTest { assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); } + @Test + public void zookeeper_logs() { + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true); + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ZERO, false); + + new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true); + assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false); + } + private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) { assertForLogFile(srcPath, destination, compression, null, rotatedOnly); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 4cf6b9e0ac1..9af37088a33 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationTransaction; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java index 21745f23177..8f06a7bb30a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.HostName; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index a73b6896c2c..169923210cb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.collections.ListMap; import com.yahoo.collections.Pair; import com.yahoo.component.AbstractComponent; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 2d36108a1a6..29642bc25dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.config.provision.Deployer; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 9a53370faf5..53defef6fc7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 64a345790cc..014c27b2ef1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index 99f1b7146df..20a1621b1d2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index 49ddfceea71..15126ed5845 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index d72c5959082..81947251e64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java index 778d34c676f..e0b8fcc5feb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 5a9d75a9d41..b3244c1ac74 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java index 577a030d658..479d6bfe079 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.orchestrator.controller; import ai.vespa.hosted.client.AbstractHttpClient; import ai.vespa.hosted.client.HttpClient; import ai.vespa.util.http.hc5.VespaHttpClientBuilder; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.vespa.applicationmodel.HostName; import org.apache.hc.core5.http.message.BasicHeader; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java index 2ebe377c238..a8a8700d631 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionRequestHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.EmptyResponse; import com.yahoo.container.jdisc.HttpResponse; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java index f54b9e319d2..7e157eade1b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthRequestHandler.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.orchestrator.resources; import ai.vespa.http.HttpURL.Path; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.RestApi; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java index 8e292d1bd6e..44a095bbc64 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.orchestrator.resources; import ai.vespa.http.HttpURL.Path; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Response; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java index eff4ab3364b..f1d7ae3857c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Response; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java index df836bf7933..16f55d9ec9b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandler.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jrt.slobrok.api.Mirror; diff --git a/parent/pom.xml b/parent/pom.xml index 125bf7bdad9..acd1ed6f25d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -108,6 +108,11 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>${maven-failsafe-plugin.version}</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> <version>${maven-install-plugin.version}</version> <configuration> @@ -953,6 +958,7 @@ <maven-dependency-plugin.version>3.3.0</maven-dependency-plugin.version> <!-- NOTE: When upgrading, also update explicit versions in tenant base poms! --> <maven-deploy-plugin.version>2.8.2</maven-deploy-plugin.version> <maven-enforcer-plugin.version>3.0.0</maven-enforcer-plugin.version> + <maven-failsafe-plugin.version>3.0.0-M6</maven-failsafe-plugin.version> <maven-install-plugin.version>3.0.0-M1</maven-install-plugin.version> <maven-jar-plugin.version>3.2.0</maven-jar-plugin.version> <maven-javadoc-plugin.version>3.3.1</maven-javadoc-plugin.version> diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClient.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClient.java index 70ed84aa014..d982bb06f32 100644 --- a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClient.java +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/status/RoutingStatusClient.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.routing.status; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.lang.CachedSupplier; import com.yahoo.routing.config.ZoneConfig; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp index a3a180a74df..3aecbc3ca0d 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/lidvectorcontext.cpp @@ -2,6 +2,7 @@ #include "lidvectorcontext.h" #include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/vespalib/objects/nbostream.h> #include <cassert> @@ -9,6 +10,7 @@ LOG_SETUP(".proton.feedoperation.lidvectorcontext"); using search::BitVector; +using search::AllocatedBitVector; namespace proton { @@ -73,7 +75,7 @@ LidVectorContext::deserialize(vespalib::nbostream &is) LOG(debug, "deserialize: format = %d", format); // Use of bitvector when > 1/32 of docs if (format == BITVECTOR) { - BitVector::UP bitVector = BitVector::create(_docIdLimit); + auto bitVector = std::make_unique<AllocatedBitVector>(_docIdLimit); is >> *bitVector; uint32_t sz(bitVector->size()); assert(sz == _docIdLimit); diff --git a/searchcore/src/vespa/searchcore/proton/matching/querylimiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.cpp index 837d1d2c2a9..df656177cbc 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querylimiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.cpp @@ -19,7 +19,7 @@ void QueryLimiter::grabToken(const Doom & doom) { std::unique_lock<std::mutex> guard(_lock); - while ((_maxThreads > 0) && (_activeThreads >= _maxThreads) && !doom.hard_doom()) { + for (auto max_threads = get_max_threads(); (max_threads > 0) && (_activeThreads >= max_threads) && !doom.hard_doom(); max_threads = get_max_threads()) { vespalib::duration left = doom.hard_left(); if (left > vespalib::duration::zero()) { _cond.wait_for(guard, left); @@ -49,18 +49,20 @@ QueryLimiter::QueryLimiter() : void QueryLimiter::configure(int maxThreads, double coverage, uint32_t minHits) { - _maxThreads = maxThreads; - _coverage = coverage; - _minHits = minHits; + std::lock_guard<std::mutex> guard(_lock); + _maxThreads.store(maxThreads, std::memory_order_relaxed); + _coverage.store(coverage, std::memory_order_relaxed); + _minHits.store(minHits, std::memory_order_relaxed); + _cond.notify_all(); } QueryLimiter::Token::UP QueryLimiter::getToken(const Doom & doom, uint32_t numDocs, uint32_t numHits, bool hasSorting, bool hasGrouping) { - if (_maxThreads > 0) { + if (get_max_threads() > 0) { if (hasSorting || hasGrouping) { - if (numHits > _minHits) { - if (numDocs * _coverage < numHits) { + if (numHits > get_min_hits()) { + if (numDocs * get_coverage() < numHits) { return std::make_unique<LimitedToken>(doom, *this); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/querylimiter.h b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.h index 57a0568b57e..67faf74a65d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querylimiter.h +++ b/searchcore/src/vespa/searchcore/proton/matching/querylimiter.h @@ -40,9 +40,13 @@ private: int _activeThreads; // These are updated asynchronously at reconfig. - volatile int _maxThreads; - volatile double _coverage; - volatile uint32_t _minHits; + std::atomic<int> _maxThreads; + std::atomic<double> _coverage; + std::atomic<uint32_t> _minHits; + + [[nodiscard]] int get_max_threads() const noexcept { return _maxThreads.load(std::memory_order_relaxed); } + [[nodiscard]] double get_coverage() const noexcept { return _coverage.load(std::memory_order_relaxed); } + [[nodiscard]] uint32_t get_min_hits() const noexcept { return _minHits.load(std::memory_order_relaxed); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index 5ade2e263c6..e010240f5f8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -180,7 +180,7 @@ class BucketMoveJob::StartMove : public storage::spi::BucketTask { public: using IDestructorCallbackSP = std::shared_ptr<vespalib::IDestructorCallback>; StartMove(std::shared_ptr<BucketMoveJob> job, BucketMover::MoveKeys keys, IDestructorCallbackSP opsTracker) - : _job(job), + : _job(std::move(job)), _keys(std::move(keys)), _opsTracker(std::move(opsTracker)) {} @@ -197,9 +197,9 @@ public: } private: - std::shared_ptr<BucketMoveJob> _job; - BucketMover::MoveKeys _keys; - IDestructorCallbackSP _opsTracker; + std::shared_ptr<BucketMoveJob> _job; + BucketMover::MoveKeys _keys; + IDestructorCallbackSP _opsTracker; }; void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 4e272e1af4c..8cc64f1b0de 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -319,7 +319,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot MaintenanceConfigSP newMaintenanceConfig; std::shared_ptr<const ThreadingServiceConfig> old_threading_service_config; std::shared_ptr<const AllocConfig> old_alloc_config; - constexpr vespalib::duration file_resolve_timeout = 60s * 10; + constexpr vespalib::duration file_resolve_timeout = 60min; if (!_ignoreForwardedConfig) { if (!(_bootstrapConfig->getDocumenttypesConfigSP() && @@ -352,6 +352,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) { newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId); } + vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) { RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP( snapshot.getConfig<RankingConstantsConfig>(_configId)); @@ -359,7 +360,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot RankingConstants::Vector constants; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const RankingConstantsConfig::Constant &rc : newRankingConstantsConfig->constant) { auto desc = fmt("name='%s', type='%s'", rc.name.c_str(), rc.type.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); @@ -375,7 +375,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot RankingExpressions expressions; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const RankingExpressionsConfig::Expression &rc : newRankingExpressionsConfig->expression) { auto desc = fmt("name='%s'", rc.name.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); @@ -391,7 +390,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot OnnxModels::Vector models; if (spec != "") { config::RpcFileAcquirer fileAcquirer(transport, spec); - vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); for (const OnnxModelsConfig::Model &rc : newOnnxModelsConfig->model) { auto desc = fmt("name='%s'", rc.name.c_str()); vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index e811e68f4b5..0658287f29b 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/vespalib/stllike/cache_stats.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/common/allocatedbitvector.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> @@ -95,7 +95,7 @@ public: uint32_t _visitCount; uint32_t _visitRmCount; uint32_t _docIdLimit; - BitVector::UP _valid; + std::unique_ptr<AllocatedBitVector> _valid; bool _before; MyVisitorBase(DocumentTypeRepo &repo, uint32_t docIdLimit, bool before); @@ -106,7 +106,7 @@ MyVisitorBase::MyVisitorBase(DocumentTypeRepo &repo, uint32_t docIdLimit, bool b _visitCount(0u), _visitRmCount(0u), _docIdLimit(docIdLimit), - _valid(BitVector::create(docIdLimit)), + _valid(std::make_unique<AllocatedBitVector>(docIdLimit)), _before(before) { } @@ -216,7 +216,7 @@ struct Fixture std::unique_ptr<LogDocumentStore> _store; uint64_t _syncToken; uint32_t _docIdLimit; - BitVector::UP _valid; + std::unique_ptr<AllocatedBitVector> _valid; Fixture(); ~Fixture(); @@ -246,7 +246,7 @@ Fixture::Fixture() _store(), _syncToken(0u), _docIdLimit(0u), - _valid(BitVector::create(0u)) + _valid(std::make_unique<AllocatedBitVector>(0u)) { rmdir(); mkdir(); diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h index 6d8c64a706d..1104d749886 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchcommon/attribute/i_multi_value_read_view.h> +#include <vespa/vespalib/util/array.h> namespace search::attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h index 1d631b06c1c..2c55452faaf 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchcommon/attribute/i_multi_value_read_view.h> +#include <vespa/vespalib/util/array.h> namespace search::attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index 8d137530dd0..d6aa136e15f 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -122,7 +122,7 @@ void FlagAttributeT<B>::setNewValues(DocId doc, const std::vector<typename B::WT BitVector * bv = _bitVectors[offset]; if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); - _bitVectorStore[offset] = BitVector::create(_bitVectorSize); + _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; ensureGuardBit(*bv); @@ -139,7 +139,7 @@ FlagAttributeT<B>::setNewBVValue(DocId doc, multivalue::ValueType_t<typename B:: BitVector * bv = _bitVectors[offset]; if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); - _bitVectorStore[offset] = BitVector::create(_bitVectorSize); + _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; ensureGuardBit(*bv); @@ -210,11 +210,11 @@ FlagAttributeT<B>::resizeBitVectors(uint32_t neededSize) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = neededSize + (neededSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); - for (BitVector * bv : _bitVectors) { + for (size_t i(0), m(_bitVectors.size()); i < m; i++) { + BitVector *bv = _bitVectors[i]; if (bv != nullptr) { - vespalib::GenerationHeldBase::UP hold(bv->grow(newSize)); + _bitVectorStore[i]->extend(newSize); ensureGuardBit(*bv); - _bitVectorHolder.hold(std::move(hold)); } } _bitVectorSize = newSize; diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index f88f46bf974..c4f4980e372 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -3,6 +3,7 @@ #include "multinumericattribute.h" #include "multi_numeric_search_context.h" +#include <vespa/searchlib/common/growablebitvector.h> namespace search { @@ -34,10 +35,10 @@ private: void removeOldGenerations(vespalib::GenerationHandler::generation_t firstUsed) override; uint32_t getOffset(int8_t value) const { return value + 128; } - vespalib::GenerationHolder _bitVectorHolder; - std::vector<std::shared_ptr<BitVector> > _bitVectorStore; - std::vector<BitVector *> _bitVectors; - uint32_t _bitVectorSize; + vespalib::GenerationHolder _bitVectorHolder; + std::vector<std::shared_ptr<GrowableBitVector> > _bitVectorStore; + std::vector<BitVector *> _bitVectors; + uint32_t _bitVectorSize; }; typedef FlagAttributeT<FlagBaseImpl> FlagAttribute; diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index ab56317db6f..f37968c70a0 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -6,10 +6,6 @@ namespace search { -using vespalib::GenerationHeldBase; -using vespalib::GenerationHeldAlloc; -using vespalib::GenerationHolder; - namespace { size_t computeCapacity(size_t capacity, size_t allocatedBytes) { @@ -131,30 +127,4 @@ AllocatedBitVector::operator=(const BitVector & rhs) return *this; } -GenerationHeldBase::UP -AllocatedBitVector::grow(Index newSize, Index newCapacity) -{ - assert(newCapacity >= newSize); - GenerationHeldBase::UP ret; - if (newCapacity != capacity()) { - AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size(), &_alloc); - if (newSize > size()) { - tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. - } - ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc); - swap(tbv); - } else { - if (newSize > size()) { - Range clearRange(size(), newSize); - setSize(newSize); - clearIntervalNoInvalidation(clearRange); - } else { - clearIntervalNoInvalidation(Range(newSize, size())); - setSize(newSize); - updateCount(); - } - } - return ret; -} - } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index 2223c7b4701..5106717dad0 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -6,6 +6,7 @@ namespace search { +class GrowableBitVector; class BitVectorTest; /** @@ -39,7 +40,7 @@ public: AllocatedBitVector(const BitVector &other); AllocatedBitVector(const AllocatedBitVector &other); - virtual ~AllocatedBitVector(); + ~AllocatedBitVector() override; AllocatedBitVector &operator=(const AllocatedBitVector &other); AllocatedBitVector &operator=(const BitVector &other); @@ -57,9 +58,7 @@ public: * * @param newLength the new length of the bit vector (in bits) */ - void resize(Index newLength) override; - - GenerationHeldBase::UP grow(Index newLength, Index newCapacity) override; + void resize(Index newLength); protected: Index _capacityBits; @@ -67,6 +66,7 @@ protected: private: friend class BitVectorTest; + friend class GrowableBitVector; void swap(AllocatedBitVector & rhs) { std::swap(_capacityBits, rhs._capacityBits); _alloc.swap(rhs._alloc); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index d5b21a38c3f..36ebc6597fa 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -2,7 +2,6 @@ #include "bitvector.h" #include "allocatedbitvector.h" -#include "growablebitvector.h" #include "partialbitvector.h" #include <vespa/searchlib/util/file_settings.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> @@ -299,18 +298,6 @@ BitVector::hasTrueBitsInternal() const } ////////////////////////////////////////////////////////////////////// -// Set new length. Destruction of content -////////////////////////////////////////////////////////////////////// -void -BitVector::resize(Index) -{ - LOG_ABORT("should not be reached"); -} -GenerationHeldBase::UP -BitVector::grow(Index, Index ) -{ - LOG_ABORT("should not be reached"); -} size_t BitVector::getFileBytes(Index bits) @@ -383,12 +370,6 @@ BitVector::create(const BitVector & rhs) return std::make_unique<AllocatedBitVector>(rhs); } -BitVector::UP -BitVector::create(Index numberOfElements, Index newCapacity, GenerationHolder &generationHolder) -{ - return std::make_unique<GrowableBitVector>(numberOfElements, newCapacity, generationHolder); -} - MMappedBitVector::MMappedBitVector(Index numberOfElements, FastOS_FileInterface &file, int64_t offset, Index doccount) : BitVector() @@ -425,7 +406,7 @@ operator<<(nbostream &out, const BitVector &bv) nbostream & -operator>>(nbostream &in, BitVector &bv) +operator>>(nbostream &in, AllocatedBitVector &bv) { uint64_t size; uint64_t cachedHits; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 02294cc1f1f..7fea95661d7 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -18,6 +18,7 @@ class FastOS_FileInterface; namespace search { class PartialBitVector; +class AllocatedBitVector; class BitVector : protected BitWord { @@ -42,7 +43,7 @@ public: bool operator == (const BitVector &right) const; const void * getStart() const { return _words; } void * getStart() { return _words; } - Index size() const { return _sz; } + Index size() const { return vespalib::atomic::load_ref_relaxed(_sz); } Index sizeBytes() const { return numBytes(getActiveSize()); } Word load_word(Index widx) const { return vespalib::atomic::load_ref_relaxed(_words[widx]); } void store_word(Index widx, Word word) { return vespalib::atomic::store_ref_relaxed(_words[widx], word); } @@ -253,11 +254,6 @@ public: return getFileBytes(size()); } - virtual void resize(Index newLength); - - virtual GenerationHeldBase::UP grow(Index newLength, Index newCapacity); - GenerationHeldBase::UP grow(Index newLength) { return grow(newLength, newLength); } - /** * This will create the appropriate vector. * @@ -271,7 +267,6 @@ public: static UP create(const BitVector & org, Index start, Index end); static UP create(Index numberOfElements); static UP create(const BitVector & rhs); - static UP create(Index newSize, Index newCapacity, GenerationHolder &generationHolder); protected: using Alloc = vespalib::alloc::Alloc; VESPA_DLL_LOCAL BitVector(void * buf, Index start, Index end); @@ -384,14 +379,14 @@ protected: friend vespalib::nbostream & operator<<(vespalib::nbostream &out, const BitVector &bv); friend vespalib::nbostream & - operator>>(vespalib::nbostream &in, BitVector &bv); + operator>>(vespalib::nbostream &in, AllocatedBitVector &bv); }; vespalib::nbostream & operator<<(vespalib::nbostream &out, const BitVector &bv); vespalib::nbostream & -operator>>(vespalib::nbostream &in, BitVector &bv); +operator>>(vespalib::nbostream &in, AllocatedBitVector &bv); template <typename T> void BitVector::andNotWithT(T it) { diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index 7c29945292e..7a548b3e7ac 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -7,8 +7,35 @@ namespace search { using vespalib::GenerationHeldBase; +using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; +GenerationHeldBase::UP +GrowableBitVector::grow(Index newSize, Index newCapacity) +{ + assert(newCapacity >= newSize); + GenerationHeldBase::UP ret; + if (newCapacity != capacity()) { + AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size(), &_alloc); + if (newSize > size()) { + tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. + } + ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc); + swap(tbv); + } else { + if (newSize > size()) { + Range clearRange(size(), newSize); + setSize(newSize); + clearIntervalNoInvalidation(clearRange); + } else { + clearIntervalNoInvalidation(Range(newSize, size())); + setSize(newSize); + updateCount(); + } + } + return ret; +} + GrowableBitVector::GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder, const Alloc* init_alloc) diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index 71261e130b6..8773f1573d6 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -16,6 +16,8 @@ public: bool shrink(Index newCapacity); bool extend(Index newCapacity); private: + GenerationHeldBase::UP grow(Index newLength, Index newCapacity); + VESPA_DLL_LOCAL bool hold(GenerationHeldBase::UP v); GenerationHolder &_generationHolder; }; diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h index 05a2e1452f6..5babd0a5814 100644 --- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h +++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h @@ -41,4 +41,20 @@ public: } }; +/** + * Feature executor that returns a constant tensor. + */ +class ConstantTensorRefExecutor : public fef::FeatureExecutor +{ +private: + const vespalib::eval::Value &_tensor_ref; +public: + ConstantTensorRefExecutor(const vespalib::eval::Value &tensor_ref) + : _tensor_ref(tensor_ref) {} + bool isPure() final override { return true; } + void execute(uint32_t) final override { + outputs().set_object(0, _tensor_ref); + } +}; + } diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index 56375eecbad..cbf7956e09b 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -25,8 +25,13 @@ using namespace search::fef; using namespace search::fef::indexproperties; using document::TensorDataType; using vespalib::eval::ValueType; +using vespalib::eval::Value; using vespalib::Issue; using search::fef::FeatureType; +using search::fef::AnyWrapper; +using search::fef::Anything; + +using ValueWrapper = AnyWrapper<Value::UP>; namespace search::features { @@ -56,6 +61,15 @@ feature_t asFeature(const vespalib::string &str) { return val; } +// query(foo): +// query.value.foo -> decoded tensor value 'foo' +vespalib::string make_value_key(const vespalib::string &base, const vespalib::string &sub_key) { + vespalib::string key(base); + key.append(".value."); + key.append(sub_key); + return key; +} + } // namespace search::features::<unnamed> QueryBlueprint::QueryBlueprint() : @@ -86,6 +100,7 @@ QueryBlueprint::setup(const IIndexEnvironment &env, const ParameterList ¶ms) _key = params[0].getValue(); _key2 = "$"; _key2.append(_key); + _stored_value_key = make_value_key(getBaseName(), _key); vespalib::string key3; key3.append("query("); @@ -114,10 +129,9 @@ QueryBlueprint::setup(const IIndexEnvironment &env, const ParameterList ¶ms) namespace { -FeatureExecutor & -createTensorExecutor(const IQueryEnvironment &env, - const vespalib::string &queryKey, - const ValueType &valueType, vespalib::Stash &stash) +Value::UP make_tensor_value(const IQueryEnvironment &env, + const vespalib::string &queryKey, + const ValueType &valueType) { Property prop = env.getProperties().lookup(queryKey); if (prop.found() && !prop.get().empty()) { @@ -125,27 +139,42 @@ createTensorExecutor(const IQueryEnvironment &env, vespalib::nbostream stream(value.data(), value.size()); try { auto tensor = vespalib::eval::decode_value(stream, vespalib::eval::FastValueBuilderFactory::get()); - if (!TensorDataType::isAssignableType(valueType, tensor->type())) { + if (TensorDataType::isAssignableType(valueType, tensor->type())) { + return tensor; + } else { Issue::report("Query feature type is '%s' but other tensor type is '%s'", valueType.to_spec().c_str(), tensor->type().to_spec().c_str()); - return ConstantTensorExecutor::createEmpty(valueType, stash); } - return ConstantTensorExecutor::create(std::move(tensor), stash); } catch (const vespalib::eval::DecodeValueException &e) { Issue::report("Query feature has invalid binary format: %s", e.what()); - return ConstantTensorExecutor::createEmpty(valueType, stash); } } - return ConstantTensorExecutor::createEmpty(valueType, stash); + return {}; +} + } +void +QueryBlueprint::prepareSharedState(const fef::IQueryEnvironment &env, fef::IObjectStore &store) const +{ + if (!_stored_value_key.empty() && _valueType.has_dimensions() && (store.get(_stored_value_key) == nullptr)) { + auto value = make_tensor_value(env, _key, _valueType); + if (value) { + store.add(_stored_value_key, std::make_unique<ValueWrapper>(std::move(value))); + } + } } FeatureExecutor & QueryBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { if (_valueType.has_dimensions()) { - return createTensorExecutor(env, _key, _valueType, stash); + if (const Anything *wrapped_value = env.getObjectStore().get(_stored_value_key)) { + if (const Value *value = ValueWrapper::getValue(*wrapped_value).get()) { + return stash.create<ConstantTensorRefExecutor>(*value); + } + } + return ConstantTensorExecutor::createEmpty(_valueType, stash); } else { std::vector<feature_t> values; Property p = env.getProperties().lookup(_key); diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.h b/searchlib/src/vespa/searchlib/features/queryfeature.h index b097f5785ed..05e44bcd923 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.h +++ b/searchlib/src/vespa/searchlib/features/queryfeature.h @@ -17,6 +17,7 @@ class QueryBlueprint : public fef::Blueprint { private: vespalib::string _key; // 'foo' vespalib::string _key2; // '$foo' + vespalib::string _stored_value_key; feature_t _defaultValue; vespalib::eval::ValueType _valueType; @@ -30,6 +31,7 @@ public: return fef::ParameterDescriptions().desc().string(); } bool setup(const fef::IIndexEnvironment &env, const fef::ParameterList ¶ms) override; + void prepareSharedState(const fef::IQueryEnvironment &env, fef::IObjectStore &store) const override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp b/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp index 6a2feaf14fa..81c1666bed6 100644 --- a/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp @@ -69,7 +69,9 @@ FeatureTest::setup() LOG(error, "Failed to compile blueprint resolver."); return false; } - + for (const auto &spec: _resolver->getExecutorSpecs()) { + spec.blueprint->prepareSharedState(_queryEnv, _queryEnv.getObjectStore()); + } _rankProgram->setup(*_match_data, _queryEnv, _overrides); _doneSetup = true; return true; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 3049b643709..427e6700e8c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -28,7 +28,7 @@ HnswGraph::make_node_for_document(uint32_t docid, uint32_t num_levels) // A document cannot be added twice. assert(!get_node_ref(docid).valid()); // Note: The level array instance lives as long as the document is present in the index. - vespalib::Array<AtomicEntryRef> levels(num_levels, AtomicEntryRef()); + std::vector<AtomicEntryRef> levels(num_levels, AtomicEntryRef()); auto node_ref = nodes.add(levels); node_refs[docid].store_release(node_ref); if (docid >= node_refs_size.load(std::memory_order_relaxed)) { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index f607af587b5..48f4cb9d494 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -17,6 +17,7 @@ #include <vespa/vespalib/datastore/entryref.h> #include <vespa/vespalib/util/rcuvector.h> #include <vespa/vespalib/util/reusable_set_pool.h> +#include <vespa/vespalib/stllike/allocator.h> namespace search::tensor { @@ -87,10 +88,9 @@ protected: using LinkStore = HnswGraph::LinkStore; using LinkArrayRef = HnswGraph::LinkArrayRef; - using LinkArray = vespalib::Array<uint32_t>; + using LinkArray = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; using LevelArrayRef = HnswGraph::LevelArrayRef; - using LevelArray = vespalib::Array<AtomicEntryRef>; using TypedCells = vespalib::eval::TypedCells; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 9215581a2b9..b8c980a8760 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index 707533d6580..64136351755 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java index dce5b6dc3d6..14b62938c20 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.manager; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index bc0db1500b8..815f8de26b4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.model; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 20ba51c337a..d59cc6c8429 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.slobrok; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index a605bdb7d39..25c4b8f801f 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -92,7 +92,6 @@ container-disc-jar-with-dependencies.jar, model-evaluation-jar-with-dependencies.jar, model-integration-jar-with-dependencies.jar, - vespajlib.jar </discPreInstallBundle> </configuration> </plugin> diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh index ca9507ae14b..a2304621845 100755 --- a/standalone-container/src/main/sh/standalone-container.sh +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -137,6 +137,10 @@ StartCommand() { cd "$VESPA_HOME" || Fail "Cannot cd to $VESPA_HOME" fixlimits + + # If JAVA_HOME is set in the environment, the sourcing of common-env.sh + # elsewhere in this file ensures $JAVA_HOME/bin is first in PATH, so 'java' + # may be invoked w/o path. In any case, checkjava verifies bare 'java'. checkjava FixDataDirectory "$(dirname "$pidfile")" diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index 9f34640228b..3c1a59dab51 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identity; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateWithKey; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index cb6ec9d1936..418f7ec024b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml index 46704533bd1..f3d94fa809d 100644 --- a/vespa-documentgen-plugin/pom.xml +++ b/vespa-documentgen-plugin/pom.xml @@ -58,6 +58,7 @@ <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-project</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>junit</groupId> diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 7ffd93f55b0..0b0e7c5b647 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -18,12 +18,11 @@ import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedStructDataType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.ApplicationBuilder; +import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.searchdefinition.parser.ParseException; import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugins.annotations.Component; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; @@ -55,7 +54,7 @@ public class DocumentGenMojo extends AbstractMojo { private static final int STD_INDENT = 4; - @Component + @Parameter( defaultValue = "${project}", readonly = true ) private MavenProject project; /** diff --git a/vespa-maven-plugin/pom.xml b/vespa-maven-plugin/pom.xml index 427c98f26d2..5984c6a1324 100644 --- a/vespa-maven-plugin/pom.xml +++ b/vespa-maven-plugin/pom.xml @@ -63,22 +63,27 @@ <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.maven.plugin-tools</groupId> <artifactId>maven-plugin-annotations</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-model</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-artifact</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> + <scope>provided</scope> </dependency> <dependency> diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index 8a0446ac2e6..90b3f972a3a 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -7,7 +7,7 @@ import ai.vespa.cloud.Zone; import ai.vespa.hosted.api.TestDescriptor; import ai.vespa.hosted.cd.InconclusiveTestException; import ai.vespa.hosted.cd.internal.TestRuntimeProvider; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.application.OsgiFramework; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java index 8408a7703be..0fdc88e1ad9 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestRunnerHandler.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.testrunner; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.EmptyResponse; import com.yahoo.container.jdisc.HttpRequest; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java index 2dbcef21718..274fa91a59c 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.testrunner; import ai.vespa.hosted.api.TestConfig; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java index 0823c2b4037..f0ea46e5220 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.testrunner.HtmlLogger; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 51707a05401..73bea9f7a00 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -3,7 +3,7 @@ package com.yahoo.document.restapi.resource; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.SystemTimer; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index bc1e58a4990..bb60b40f058 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.restapi.resource; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 6044666ebf8..f7be61946ba 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -787,7 +787,10 @@ "public", "abstract" ], - "methods": [], + "methods": [ + "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(float[])", + "public com.yahoo.tensor.IndexedTensor$BoundBuilder fill(double[])" + ], "fields": [] }, "com.yahoo.tensor.IndexedTensor$Builder": { @@ -903,7 +906,8 @@ "public java.util.Map cells()", "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public bridge synthetic com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)" ], @@ -954,7 +958,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)" ], "fields": [] @@ -1046,7 +1051,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public long denseSubspaceSize()", "public static com.yahoo.tensor.TensorType createPartialType(com.yahoo.tensor.TensorType$Value, java.util.List)" @@ -1092,6 +1098,7 @@ ], "methods": [ "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, java.lang.String)", + "public com.yahoo.tensor.TensorType type()", "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, long)", "public com.yahoo.tensor.Tensor$Builder value(double)", "public com.yahoo.tensor.Tensor$Builder value(float)" @@ -1233,9 +1240,11 @@ "public java.util.List largest()", "public java.util.List smallest()", "public abstract java.lang.String toString()", - "public abstract java.lang.String toShortString()", - "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, long)", - "public static java.lang.String contentToString(com.yahoo.tensor.Tensor, long)", + "public abstract java.lang.String toString(boolean, boolean)", + "public abstract java.lang.String toAbbreviatedString()", + "public java.lang.String toShortString()", + "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, boolean, boolean, long)", + "public static java.lang.String valueToString(com.yahoo.tensor.Tensor, boolean, long)", "public abstract boolean equals(java.lang.Object)", "public abstract int hashCode()", "public static boolean equals(com.yahoo.tensor.Tensor, com.yahoo.tensor.Tensor)", @@ -1256,8 +1265,10 @@ ], "methods": [ "public void <init>(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String)", "public com.yahoo.tensor.TensorAddress$Builder add(java.lang.String, java.lang.String)", "public com.yahoo.tensor.TensorAddress$Builder copy()", + "public com.yahoo.tensor.TensorType type()", "public com.yahoo.tensor.TensorAddress build()" ], "fields": [] diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 89eefeced56..c4316eb334a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -219,21 +219,26 @@ public abstract class IndexedTensor implements Tensor { } @Override - public String toString() { return toString(Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } - Indexes indexes = Indexes.of(dimensionSizes); + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms || type.rank() == 0 || type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - StringBuilder b = new StringBuilder(type.toString()).append(":"); + Indexes indexes = Indexes.of(dimensionSizes); + StringBuilder b = new StringBuilder(); + if (withType) + b.append(type).append(":"); indexedBlockToString(this, indexes, maxCells, b); return b.toString(); } @@ -438,7 +443,7 @@ public abstract class IndexedTensor implements Tensor { this.sizes = sizes; } - BoundBuilder fill(float[] values) { + public BoundBuilder fill(float[] values) { long index = 0; for (float value : values) { cellByDirectIndex(index++, value); @@ -446,7 +451,7 @@ public abstract class IndexedTensor implements Tensor { return this; } - BoundBuilder fill(double[] values) { + public BoundBuilder fill(double[] values) { long index = 0; for (double value : values) { cellByDirectIndex(index++, value); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index ad945ed18bf..946d8fe0f4a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -72,11 +72,18 @@ public class MappedTensor implements Tensor { public int hashCode() { return cells.hashCode(); } @Override - public String toString() { return Tensor.toStandardString(this, Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return Tensor.toStandardString(this, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { return toString(withType, shortForms, Long.MAX_VALUE); } + + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + return Tensor.toStandardString(this, withType, shortForms, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 56bd94a86e9..d2fed9b96f9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -145,23 +145,27 @@ public class MixedTensor implements Tensor { @Override public String toString() { - return toString(Long.MAX_VALUE); + return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) - return Tensor.toStandardString(this, maxCells); - if (type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms + || type.rank() == 0 + || type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty()) + || type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - return type + ":" + index.contentToString(this, maxCells); + return (withType ? type + ":" : "") + index.contentToString(this, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 06e7b010a7a..8a84e97fe05 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -32,7 +32,6 @@ import java.util.Set; import java.util.function.DoubleBinaryOperator; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; -import java.util.stream.Collectors; import static com.yahoo.tensor.functions.ScalarFunctions.Hamming; @@ -316,8 +315,22 @@ public interface Tensor { @Override String toString(); + /** + * Returns this tensor on the + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * + * @param withType whether to prefix the value by the type of this + * @param shortForms whether to use short forms where applicable, or always using the verbose form + */ + String toString(boolean withType, boolean shortForms); + /** Returns an abbreviated string representation of this tensor suitable for human-readable messages */ - String toShortString(); + String toAbbreviatedString(); + + // TODO: Remove on Vespa 8 + /** @deprecated use toAbbreviatedString */ + @Deprecated + default String toShortString() { return toAbbreviatedString(); } /** * Call this from toString in implementations to return this tensor on the @@ -325,15 +338,16 @@ public interface Tensor { * (toString cannot be a default method because default methods cannot override super methods). * * @param tensor the tensor to return the standard string format of + * @param withType whether the type should be prepended to the content * @param maxCells the max number of cells to output, after which just , "..." is output to represent the rest * of the cells * @return the tensor on the standard string format */ - static String toStandardString(Tensor tensor, long maxCells) { - return tensor.type() + ":" + contentToString(tensor, maxCells); + static String toStandardString(Tensor tensor, boolean withType, boolean shortForms, long maxCells) { + return (withType ? tensor.type() + ":" : "") + valueToString(tensor, shortForms, maxCells); } - static String contentToString(Tensor tensor, long maxCells) { + static String valueToString(Tensor tensor, boolean shortForms, long maxCells) { var cellEntries = new ArrayList<>(tensor.cells().entrySet()); cellEntries.sort(Map.Entry.comparingByKey()); if (tensor.type().dimensions().isEmpty()) { @@ -345,7 +359,7 @@ public interface Tensor { for (; i < cellEntries.size() && i < maxCells; i++) { if (i > 0) b.append(", "); - b.append(cellToString(cellEntries.get(i), tensor.type())); + b.append(cellToString(cellEntries.get(i), tensor.type(), shortForms)); } if (i == maxCells && i < tensor.size()) b.append(", ..."); @@ -353,8 +367,9 @@ public interface Tensor { return b.toString(); } - private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type) { - return (type.rank() > 1 ? cell.getKey().toString(type) : TensorAddress.labelToString(cell.getKey().label(0))) + + private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type, boolean shortForms) { + return (shortForms && type.rank() == 1 ? TensorAddress.labelToString(cell.getKey().label(0)) + : cell.getKey().toString(type) ) + ":" + cell.getValue(); } @@ -533,7 +548,7 @@ public interface Tensor { return IndexedTensor.Builder.of(type, dimensionSizes); } - /** Returns the type this is building */ + /** Returns the type of the tensor this is building */ TensorType type(); /** Return a cell builder */ @@ -578,6 +593,9 @@ public interface Tensor { return this; } + /** Returns the type of the tensor this cell is build for. */ + public TensorType type() { return tensorBuilder.type(); } + public CellBuilder label(String dimension, long label) { return label(dimension, String.valueOf(label)); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 27e752f1180..d9ab67d6c5f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -168,7 +168,7 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } - /** Supports building of a tensor address */ + /** Builder of a tensor address */ public static class Builder { private final TensorType type; @@ -184,6 +184,18 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } /** + * Adds the label to the only dimension of this. + * + * @throws IllegalArgumentException if this does not have exactly one dimension + */ + public Builder add(String label) { + if (type.rank() != 1) + throw new IllegalArgumentException("Cannot add a label without explicit dimension to a tensor of type " + type); + add(type.dimensions().get(0).name(), label); + return this; + } + + /** * Adds a label in a dimension to this. * * @return this for convenience @@ -203,6 +215,9 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { return new Builder(type, Arrays.copyOf(labels, labels.length)); } + /** Returns the type of the tensor this address is being built for. */ + public TensorType type() { return type; } + public TensorAddress build() { for (int i = 0; i < labels.length; i++) if (labels[i] == null) diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 2067d7a8492..920f8512c53 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -65,24 +65,24 @@ public class TensorTestCase { @Test public void testToShortString() { assertEquals("tensor(x[10]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", - Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toShortString()); + Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toAbbreviatedString()); assertEquals("tensor(x[14]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, ...]", - Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toShortString()); + Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toAbbreviatedString()); assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0, {d1:l1,d2:l3}:6.0, ...}", Tensor.from("{{d1:l1,d2:l1}:6, {d2:l2,d1:l1}:6, {d2:l3,d1:l1}:6, {d2:l4,d1:l1}:6, {d2:l5,d1:l1}:6," + " {d2:l6,d1:l1}:6, {d2:l7,d1:l1}:6, {d2:l8,d1:l1}:6, {d2:l9,d1:l1}:6, {d2:l2,d1:l2}:6," + - " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toShortString()); + " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},n{},x[3]):{{m:k1,n:k1,x:0}:0.0, {m:k1,n:k1,x:1}:1.0, {m:k1,n:k1,x:2}:2.0, ...}", Tensor.from("tensor(m{},n{},x[3]):" + "{{m:k1,n:k1,x:0}:0, {m:k1,n:k1,x:1}:1, {m:k1,n:k1,x:2}:2, " + " {m:k2,n:k1,x:0}:0, {m:k2,n:k1,x:1}:1, {m:k2,n:k1,x:2}:2, " + - " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toShortString()); + " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toAbbreviatedString()); assertEquals("tensor(m{},x[2],y[2]):{k1:[[0.0, 1.0], [2.0, 3.0]], k2:[[0.0, ...}", - Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toShortString()); + Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toAbbreviatedString()); } @Test diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp index ca43027b2b5..c4af25efcac 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp +++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp @@ -41,7 +41,7 @@ nbostream::nbostream(const void * buf, size_t sz) : nbostream::nbostream(Alloc && buf, size_t sz) : _wbuf(std::move(buf), sz), - _rbuf(&_wbuf[0], sz), + _rbuf(_wbuf.data(), sz), _rp(0), _wp(sz), _state(ok), @@ -60,7 +60,7 @@ nbostream::nbostream(const nbostream & rhs) : { extend(rhs.size()); _wp = rhs.size(); - memcpy(&_wbuf[0], &rhs._rbuf[rhs._rp], _wp); + memcpy(_wbuf.data(), &rhs._rbuf[rhs._rp], _wp); } nbostream::nbostream(nbostream && rhs) noexcept @@ -76,10 +76,10 @@ nbostream::nbostream(nbostream && rhs) noexcept rhs._rbuf = ConstBufferRef(); if (!_longLivedBuffer && (_wbuf.capacity() == 0)) { _wbuf.resize(roundUp2inN(_rbuf.size())); - memcpy(&_wbuf[0], &_rbuf[_rp], size()); + memcpy(_wbuf.data(), &_rbuf[_rp], size()); _wp = size(); _rp = 0; - _rbuf = ConstBufferRef(&_wbuf[0], _wbuf.capacity()); + _rbuf = ConstBufferRef(_wbuf.data(), _wbuf.capacity()); } } @@ -120,7 +120,7 @@ void nbostream::reserve(size_t sz) void nbostream::compact() { - memmove(&_wbuf[0], &_rbuf[_rp], left()); + memmove(_wbuf.data(), &_rbuf[_rp], left()); _wp = left(); _rp = 0; } @@ -148,7 +148,7 @@ void nbostream::swap(Buffer & buf) { } _wbuf.resize(size()); _wbuf.swap(buf); - _rbuf = ConstBufferRef(&_wbuf[0], _wbuf.capacity()); + _rbuf = ConstBufferRef(_wbuf.data(), _wbuf.capacity()); _wp = _wbuf.size(); _rp = 0; _state = ok; diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h index 337833d2457..88dc501370c 100644 --- a/vespalib/src/vespa/vespalib/util/arrayref.h +++ b/vespalib/src/vespa/vespalib/util/arrayref.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "array.h" #include "small_vector.h" #include <vector> @@ -14,19 +13,20 @@ namespace vespalib { template <typename T> class ArrayRef { public: - ArrayRef() noexcept : _v(nullptr), _sz(0) { } - ArrayRef(T * v, size_t sz) noexcept : _v(v), _sz(sz) { } + constexpr ArrayRef() noexcept : _v(nullptr), _sz(0) { } + constexpr ArrayRef(T * v, size_t sz) noexcept : _v(v), _sz(sz) { } template<typename A=std::allocator<T>> - ArrayRef(std::vector<T, A> & v) noexcept : _v(&v[0]), _sz(v.size()) { } + ArrayRef(std::vector<T, A> & v) noexcept : _v(v.data()), _sz(v.size()) { } template<size_t N> - ArrayRef(SmallVector<T, N> &v) noexcept : _v(&v[0]), _sz(v.size()) { } - ArrayRef(Array<T> &v) noexcept : _v(&v[0]), _sz(v.size()) { } - T & operator [] (size_t i) { return _v[i]; } - const T & operator [] (size_t i) const { return _v[i]; } - size_t size() const { return _sz; } - bool empty() const { return _sz == 0; } - T *begin() { return _v; } - T *end() { return _v + _sz; } + ArrayRef(SmallVector<T, N> &v) noexcept : _v(v.data()), _sz(v.size()) { } + T & operator [] (size_t i) noexcept { return _v[i]; } + const T & operator [] (size_t i) const noexcept { return _v[i]; } + T * data() noexcept { return _v; } + const T * data() const noexcept { return _v; } + [[nodiscard]] size_t size() const noexcept { return _sz; } + [[nodiscard]] bool empty() const noexcept { return _sz == 0; } + T *begin() noexcept { return _v; } + T *end() noexcept { return _v + _sz; } private: T * _v; size_t _sz; @@ -35,21 +35,20 @@ private: template <typename T> class ConstArrayRef { public: - ConstArrayRef(const T *v, size_t sz) noexcept : _v(v), _sz(sz) { } + constexpr ConstArrayRef(const T *v, size_t sz) noexcept : _v(v), _sz(sz) { } template<typename A=std::allocator<T>> - ConstArrayRef(const std::vector<T, A> & v) noexcept : _v(&v[0]), _sz(v.size()) { } + ConstArrayRef(const std::vector<T, A> & v) noexcept : _v(v.data()), _sz(v.size()) { } template<size_t N> - ConstArrayRef(const SmallVector<T, N> &v) noexcept : _v(&v[0]), _sz(v.size()) { } - ConstArrayRef(const ArrayRef<T> & v) noexcept : _v(&v[0]), _sz(v.size()) { } - ConstArrayRef(const Array<T> &v) noexcept : _v(&v[0]), _sz(v.size()) { } + ConstArrayRef(const SmallVector<T, N> &v) noexcept : _v(v.data()), _sz(v.size()) { } + ConstArrayRef(const ArrayRef<T> & v) noexcept : _v(v.data()), _sz(v.size()) { } constexpr ConstArrayRef() noexcept : _v(nullptr), _sz(0) {} - const T & operator [] (size_t i) const { return _v[i]; } - size_t size() const { return _sz; } - bool empty() const { return _sz == 0; } - const T *cbegin() const { return _v; } - const T *cend() const { return _v + _sz; } - const T *begin() const { return _v; } - const T *end() const { return _v + _sz; } + const T & operator [] (size_t i) const noexcept { return _v[i]; } + [[nodiscard]] size_t size() const noexcept { return _sz; } + [[nodiscard]] bool empty() const noexcept { return _sz == 0; } + const T *cbegin() const noexcept { return _v; } + const T *cend() const noexcept { return _v + _sz; } + const T *begin() const noexcept { return _v; } + const T *end() const noexcept { return _v + _sz; } const T *data() const noexcept { return _v; } private: const T *_v; @@ -59,7 +58,7 @@ private: // const-cast for array references; use with care template <typename T> ArrayRef<T> unconstify(const ConstArrayRef<T> &ref) { - return ArrayRef<T>(const_cast<T*>(&ref[0]), ref.size()); + return ArrayRef<T>(const_cast<T*>(ref.data()), ref.size()); } } diff --git a/vespalib/src/vespa/vespalib/util/small_vector.h b/vespalib/src/vespa/vespalib/util/small_vector.h index c1549ef725d..cf9910e24b5 100644 --- a/vespalib/src/vespa/vespalib/util/small_vector.h +++ b/vespalib/src/vespa/vespalib/util/small_vector.h @@ -4,10 +4,8 @@ #include "alloc.h" #include "traits.h" -#include <string.h> -#include <cstdint> +#include <cstring> #include <cassert> -#include <memory> #include <iterator> namespace vespalib { @@ -186,6 +184,8 @@ public: const T *end() const { return (_data + _size); } T &operator[](size_t idx) { return _data[idx]; } const T &operator[](size_t idx) const { return _data[idx]; } + T *data() noexcept { return _data; } + const T *data() const noexcept { return _data; } T &back() { return _data[_size - 1]; } const T &back() const { return _data[_size - 1]; } void clear() { diff --git a/vespalog/pom.xml b/vespalog/pom.xml index aab362e3395..1fa007d6029 100644 --- a/vespalog/pom.xml +++ b/vespalog/pom.xml @@ -22,16 +22,19 @@ <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index ae10502e569..3e1198a4e3e 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.DaemonThreadFactory; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index 7c91e54dd4b..e61e4f3add9 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.mock; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.path.Path; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index 246911fdfc7..e94110af2fb 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.AbstractComponent; diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index 8f3a5a91a43..48f95d28910 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.AbstractComponent; diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java index 2e0fcae2007..b8bd50a07fe 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/DummyVespaZooKeeperServer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import java.nio.file.Path; diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java index 46b7f8fa0ca..9d27b103b6e 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; -import com.google.inject.Inject; +import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.net.HostName; |