diff options
306 files changed, 4373 insertions, 2091 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java index 43f161cfec9..417b5792586 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java @@ -81,12 +81,26 @@ public class ServiceCluster { Objects.equals(serviceType, ServiceType.HOST_ADMIN); } + public boolean isConfigServerHostLike() { + return isConfigServerHost() || isControllerHost(); + } + public boolean isTenantHost() { return isHostedVespaApplicationWithPredicate(ApplicationInstanceId::isTenantHost) && Objects.equals(clusterId, ClusterId.TENANT_HOST) && Objects.equals(serviceType, ServiceType.HOST_ADMIN); } + public String nodeDescription(boolean plural) { + String pluralSuffix = plural ? "s" : ""; + return isConfigServer() ? "config server" + pluralSuffix : + isConfigServerHost() ? "config server host" + pluralSuffix : + isController() ? "controller" + pluralSuffix : + isControllerHost() ? "controller host" + pluralSuffix : + isTenantHost() ? "tenant host" + pluralSuffix : + "node" + pluralSuffix + " of {" + serviceType + "," + clusterId + "}"; + } + private boolean isHostedVespaApplicationWithId(ApplicationInstanceId id) { return isHostedVespaTenant() && applicationInstance.map(app -> Objects.equals(app.applicationInstanceId(), id)).orElse(false); diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 0cc3f04a57b..855b3afafaf 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -49,16 +49,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>node-repository</artifactId> <version>${project.version}</version> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CkmsKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java index bc044f12b15..88603dff57d 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CkmsKeyProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/CkmsKeyProvider.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; import com.yahoo.config.provision.Zone; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java index b2ae42cc294..3ea8eb1f538 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGenerator.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; import com.yahoo.config.provision.Zone; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java new file mode 100644 index 00000000000..8593401b887 --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java @@ -0,0 +1,98 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.RestApi; +import com.yahoo.restapi.RestApiException; +import com.yahoo.restapi.RestApiRequestHandler; +import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; +import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; + +import java.util.logging.Level; + +/** + * Handler implementing the Athenz Identity Provider API (Copper Argos). + * + * @author bjorncs + */ +public class IdentityProviderRequestHandler extends RestApiRequestHandler<IdentityProviderRequestHandler> { + + private final IdentityDocumentGenerator documentGenerator; + private final InstanceValidator instanceValidator; + + @Inject + public IdentityProviderRequestHandler(LoggingRequestHandler.Context context, + IdentityDocumentGenerator documentGenerator, + InstanceValidator instanceValidator) { + super(context, IdentityProviderRequestHandler::createRestApi); + this.documentGenerator = documentGenerator; + this.instanceValidator = instanceValidator; + } + + private static RestApi createRestApi(IdentityProviderRequestHandler self) { + return RestApi.builder() + .addRoute(RestApi.route("/athenz/v1/provider/identity-document/node/{host}") + .get(self::getNodeIdentityDocument)) + .addRoute(RestApi.route("/athenz/v1/provider/identity-document/tenant/{host}") + .get(self::getTenantIdentityDocument)) + .addRoute(RestApi.route("/athenz/v1/provider/instance") + .post(self::confirmInstance)) + .addRoute(RestApi.route("/athenz/v1/provider/refresh") + .post(self::confirmInstanceRefresh)) + // Overriding object mapper to change serialization of timestamps + .setObjectMapper(new ObjectMapper() + .registerModule(new JavaTimeModule()) + .registerModule(new Jdk8Module()) + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)) + .build(); + } + + private SignedIdentityDocumentEntity getNodeIdentityDocument(RestApi.RequestContext context) { + String host = context.pathParameters().getString("host").orElse(null); + return getIdentityDocument(host, IdentityType.NODE); + } + + private SignedIdentityDocumentEntity getTenantIdentityDocument(RestApi.RequestContext context) { + String host = context.pathParameters().getString("host").orElse(null); + return getIdentityDocument(host, IdentityType.TENANT); + } + + private InstanceConfirmation confirmInstance(RestApi.RequestContext context) { + InstanceConfirmation instanceConfirmation = context.requestContentOrThrow().consumeJacksonEntity(InstanceConfirmation.class); + log.log(Level.FINE, instanceConfirmation.toString()); + if (!instanceValidator.isValidInstance(instanceConfirmation)) { + log.log(Level.SEVERE, "Invalid instance: " + instanceConfirmation); + throw new RestApiException.Forbidden("Instance is invalid"); + } + return instanceConfirmation; + } + + private InstanceConfirmation confirmInstanceRefresh(RestApi.RequestContext context) { + InstanceConfirmation instanceConfirmation = context.requestContentOrThrow().consumeJacksonEntity(InstanceConfirmation.class); + log.log(Level.FINE, instanceConfirmation.toString()); + if (!instanceValidator.isValidRefresh(instanceConfirmation)) { + log.log(Level.SEVERE, "Invalid instance refresh: " + instanceConfirmation); + throw new RestApiException.Forbidden("Instance is invalid"); + } + return instanceConfirmation; + } + + private SignedIdentityDocumentEntity getIdentityDocument(String hostname, IdentityType identityType) { + if (hostname == null) { + throw new RestApiException.BadRequest("The 'hostname' query parameter is missing"); + } + try { + return EntityBindingsMapper.toSignedIdentityDocumentEntity(documentGenerator.generateSignedIdentityDocument(hostname, identityType)); + } catch (Exception e) { + String message = String.format("Unable to generate identity document for '%s': %s", hostname, e.getMessage()); + log.log(Level.SEVERE, message, e); + throw new RestApiException.InternalServerError(message, e); + } + } +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmation.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java index e6dd40faaca..27507c425cd 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmation.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; @@ -13,8 +13,8 @@ import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.yahoo.restapi.RestApi; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; import java.io.IOException; import java.util.HashMap; @@ -26,7 +26,7 @@ import java.util.Objects; * * @author bjorncs */ -public class InstanceConfirmation { +public class InstanceConfirmation implements RestApi.JacksonRequestEntity, RestApi.JacksonResponseEntity { @JsonProperty("provider") public final String provider; @JsonProperty("domain") public final String domain; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java index 9c5abb791cf..3dcb5a13d6d 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidator.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.common.net.InetAddresses; import com.google.inject.Inject; @@ -7,14 +7,11 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; - -import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -24,6 +21,7 @@ import java.security.PublicKey; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java index f52493375f1..4ee64044da8 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/Utils.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java deleted file mode 100644 index cb0b1e0557f..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; - -import com.google.inject.Inject; -import com.yahoo.container.jaxrs.annotation.Component; -import java.util.logging.Level; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentApi; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; - -import javax.ws.rs.BadRequestException; -import javax.ws.rs.GET; -import javax.ws.rs.InternalServerErrorException; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import java.util.logging.Logger; - -/** - * An API that issues signed identity documents for Vespa nodes. - * - * @author bjorncs - */ -@Path("/identity-document") -public class IdentityDocumentResource implements IdentityDocumentApi { - - private static final Logger log = Logger.getLogger(IdentityDocumentResource.class.getName()); - - private final IdentityDocumentGenerator identityDocumentGenerator; - - @Inject - public IdentityDocumentResource(@Component IdentityDocumentGenerator identityDocumentGenerator) { - this.identityDocumentGenerator = identityDocumentGenerator; - } - - private SignedIdentityDocumentEntity getIdentityDocument(String hostname, IdentityType identityType) { - if (hostname == null) { - throw new BadRequestException("The 'hostname' query parameter is missing"); - } - try { - return EntityBindingsMapper.toSignedIdentityDocumentEntity(identityDocumentGenerator.generateSignedIdentityDocument(hostname, identityType)); - } catch (Exception e) { - String message = String.format("Unable to generate identity doument for '%s': %s", hostname, e.getMessage()); - log.log(Level.SEVERE, message, e); - throw new InternalServerErrorException(message, e); - } - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - @Path("/node/{host}") - @Override - public SignedIdentityDocumentEntity getNodeIdentityDocument(@PathParam("host") String host) { - return getIdentityDocument(host, IdentityType.NODE); - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - @Path("/tenant/{host}") - @Override - public SignedIdentityDocumentEntity getTenantIdentityDocument(@PathParam("host") String host) { - return getIdentityDocument(host, IdentityType.TENANT); - } - -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java deleted file mode 100644 index dbe26d3a6bb..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; - -import com.google.inject.Inject; -import com.yahoo.container.jaxrs.annotation.Component; -import java.util.logging.Level; - -import javax.ws.rs.Consumes; -import javax.ws.rs.ForbiddenException; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import java.util.logging.Logger; - -/** - * @author bjorncs - */ -@Path("/instance") -public class InstanceConfirmationResource { - - private static final Logger log = Logger.getLogger(InstanceConfirmationResource.class.getName()); - - private final InstanceValidator instanceValidator; - - @Inject - public InstanceConfirmationResource(@Component InstanceValidator instanceValidator) { - this.instanceValidator = instanceValidator; - } - - @POST - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - public InstanceConfirmation confirmInstance(InstanceConfirmation instanceConfirmation) { - log.log(Level.FINE, instanceConfirmation.toString()); - if (!instanceValidator.isValidInstance(instanceConfirmation)) { - log.log(Level.SEVERE, "Invalid instance: " + instanceConfirmation); - throw new ForbiddenException("Instance is invalid"); - } - return instanceConfirmation; - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java deleted file mode 100644 index b49e03658e6..00000000000 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; - -import com.google.inject.Inject; -import com.yahoo.container.jaxrs.annotation.Component; -import java.util.logging.Level; - -import javax.ws.rs.Consumes; -import javax.ws.rs.ForbiddenException; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import java.util.logging.Logger; - -/** - * ZTS calls this resource when it's requested to refresh an instance certificate - * - * @author bjorncs - */ -@Path("/refresh") -public class InstanceRefreshResource { - - private static final Logger log = Logger.getLogger(InstanceRefreshResource.class.getName()); - - private final InstanceValidator instanceValidator; - - @Inject - public InstanceRefreshResource(@Component InstanceValidator instanceValidator) { - this.instanceValidator = instanceValidator; - } - - @POST - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - public InstanceConfirmation confirmInstanceRefresh(InstanceConfirmation instanceConfirmation) { - log.log(Level.FINE, instanceConfirmation.toString()); - if (!instanceValidator.isValidRefresh(instanceConfirmation)) { - log.log(Level.SEVERE, "Invalid instance refresh: " + instanceConfirmation); - throw new ForbiddenException("Instance is invalid"); - } - return instanceConfirmation; - } -} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 0325971038f..c9850b70afd 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -19,8 +19,8 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceConfirmation; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; import com.yahoo.vespa.hosted.ca.Certificates; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; @@ -37,7 +37,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Function; -import java.util.logging.Level; import java.util.stream.Stream; /** diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java index c409cf4f054..74289c7a451 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityDocumentGeneratorTest.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -17,7 +17,6 @@ import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.AutoGeneratedKeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java index f2f76e58142..cde63c6a0cb 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation; +package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; @@ -19,7 +19,6 @@ import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -41,8 +40,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator.SERVICE_PROPERTIES_DOMAIN_KEY; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator.SERVICE_PROPERTIES_SERVICE_KEY; +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.SERVICE_PROPERTIES_DOMAIN_KEY; +import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator.SERVICE_PROPERTIES_SERVICE_KEY; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java index 9c1d4c49b07..9d7809740bd 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java @@ -1,8 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.ca.restapi.mock; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceConfirmation; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; /** * @author mortent diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index ee3ed48318b..ddddab4bf90 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -35,7 +35,7 @@ <junit5.platform.version>1.7.0</junit5.platform.version> <org.lz4.version>1.7.1</org.lz4.version> <org.json.version>20090211</org.json.version><!-- TODO Vespa 8: remove as provided dependency --> - <slf4j.version>1.7.5</slf4j.version> + <slf4j.version>1.7.30</slf4j.version> <tensorflow.version>1.12.0</tensorflow.version> <xml-apis.version>1.4.01</xml-apis.version> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index ed8e39347e5..2a07f9ac300 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -209,7 +209,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd public boolean isMaster() { synchronized (monitor) { - return masterElectionHandler.isMaster(); + return isMaster; } } @@ -387,7 +387,9 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd // Iff master, always store new version in ZooKeeper _before_ publishing to any // nodes so that a cluster controller crash after publishing but before a successful // ZK store will not risk reusing the same version number. - if (masterElectionHandler.isMaster()) { + // Use isMaster instead of election handler state, as isMaster is set _after_ we have + // completed a leadership event edge, so we know we have read from ZooKeeper. + if (isMaster) { storeClusterStateMetaDataToZooKeeper(stateBundle); } } @@ -439,7 +441,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd */ public void lostDatabaseConnection() { verifyInControllerThread(); - boolean wasMaster = masterElectionHandler.isMaster(); + boolean wasMaster = isMaster; masterElectionHandler.lostDatabaseConnection(); if (wasMaster) { // Enforce that we re-fetch all state information from ZooKeeper upon the next tick if we're still master. @@ -522,6 +524,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd masterElectionHandler.setFleetControllerCount(options.fleetControllerCount); masterElectionHandler.setMasterZooKeeperCooldownPeriod(options.masterZooKeeperCooldownPeriod); + masterElectionHandler.setUsingZooKeeper(options.zooKeeperServerAddress != null && !options.zooKeeperServerAddress.isEmpty()); if (rpcServer != null) { rpcServer.setMasterElectionHandler(masterElectionHandler); @@ -618,7 +621,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd if ( ! isRunning()) { return; } didWork |= systemStateBroadcaster.processResponses(); if ( ! isRunning()) { return; } - if (masterElectionHandler.isMaster()) { + if (isMaster) { didWork |= broadcastClusterStateToEligibleNodes(); systemStateBroadcaster.checkIfClusterStateIsAckedByAllDistributors(database, databaseContext, this); } @@ -776,7 +779,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd private boolean taskMayBeCompletedImmediately(RemoteClusterControllerTask task) { // We cannot introduce a version barrier for tasks when we're not the master (and therefore will not publish new versions). - return (!task.hasVersionAckDependency() || task.isFailed() || !masterElectionHandler.isMaster()); + return (!task.hasVersionAckDependency() || task.isFailed() || !isMaster); } private RemoteClusterControllerTask.Context createRemoteTaskProcessingContext() { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 6e968fef7ce..2c03520ec01 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -25,6 +25,7 @@ public class MasterElectionHandler implements MasterInterface { private Map<Integer, Integer> nextMasterData; private long masterGoneFromZooKeeperTime; // Set to time master fleet controller disappears from zookeeper private long masterZooKeeperCooldownPeriod; // The period in ms that we won't take over unless master come back. + private boolean usingZooKeeper = false; // Unit tests may not use ZooKeeper at all. public MasterElectionHandler(int index, int totalCount, Object monitor, Timer timer) { this.monitor = monitor; @@ -42,7 +43,7 @@ public class MasterElectionHandler implements MasterInterface { public void setFleetControllerCount(int count) { totalCount = count; - if (count == 1) { + if (count == 1 && !usingZooKeeper) { masterCandidate = 0; followers = 1; nextInLineCount = 0; @@ -53,6 +54,14 @@ public class MasterElectionHandler implements MasterInterface { masterZooKeeperCooldownPeriod = period; } + public void setUsingZooKeeper(boolean usingZK) { + if (!usingZooKeeper && usingZK) { + // Reset any shortcuts taken by non-ZK election logic. + resetElectionProgress(); + } + usingZooKeeper = usingZK; + } + @Override public boolean isMaster() { Integer master = getMaster(); @@ -111,7 +120,9 @@ public class MasterElectionHandler implements MasterInterface { public boolean watchMasterElection(DatabaseHandler database, DatabaseHandler.Context dbContext) throws InterruptedException { - if (totalCount == 1) return false; // No point in doing master election with only one node configured to be cluster controller + if (totalCount == 1 && !usingZooKeeper) { + return false; // Allow single configured node to become master implicitly if no ZK configured + } if (nextMasterData == null) { if (masterCandidate == null) { log.log(Level.FINEST, "Cluster controller " + index + ": No current master candidate. Waiting for data to do master election."); @@ -222,15 +233,19 @@ public class MasterElectionHandler implements MasterInterface { } public void lostDatabaseConnection() { - if (totalCount > 1) { + if (totalCount > 1 || usingZooKeeper) { log.log(Level.INFO, "Cluster controller " + index + ": Clearing master data as we lost connection on node " + index); - masterData = null; - masterCandidate = null; - followers = 0; - nextMasterData = null; + resetElectionProgress(); } } + private void resetElectionProgress() { + masterData = null; + masterCandidate = null; + followers = 0; + nextMasterData = null; + } + public void writeHtmlState(StringBuilder sb, int stateGatherCount) { sb.append("<h2>Master state</h2>\n"); Integer master = getMaster(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java index ceb81e91b7d..65b97a3ae82 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java @@ -12,4 +12,11 @@ public interface NodeLookup { boolean updateCluster(ContentCluster cluster, NodeAddedOrRemovedListener listener); + /** + * Returns whether the lookup instance has been able to bootstrap itself with information about nodes. + * + * Calling updateCluster() _before_ isReady has returned true may not provide any useful data. + */ + boolean isReady(); + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index 3f04bbd9200..d19425a7c95 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -91,6 +91,7 @@ public class DatabaseHandler { private long lastZooKeeperConnectionAttempt = 0; private static final int minimumWaitBetweenFailedConnectionAttempts = 10000; private boolean lostZooKeeperConnectionEvent = false; + private boolean connectionEstablishmentIsAllowed = false; private Map<Integer, Integer> masterDataEvent = null; public DatabaseHandler(DatabaseFactory databaseFactory, Timer timer, String zooKeeperAddress, int ourIndex, Object monitor) throws InterruptedException diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index b3bb458ed74..8649e7cc11a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -59,6 +59,7 @@ public class SlobrokClient implements NodeLookup { freshMirror = true; } + @Override public void shutdown() { if (supervisor != null) { supervisor.transport().shutdown().join(); @@ -67,6 +68,12 @@ public class SlobrokClient implements NodeLookup { public Mirror getMirror() { return mirror; } + @Override + public boolean isReady() { + return mirror != null && mirror.ready(); + } + + @Override public boolean updateCluster(ContentCluster cluster, NodeAddedOrRemovedListener listener) { if (mirror == null) return false; int mirrorVersion = mirror.updates(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java index 9c0a94309a5..f6add77423a 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java @@ -10,6 +10,7 @@ import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java index b322c62967a..d7bca47026f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java @@ -154,4 +154,8 @@ public class DummyCommunicator implements Communicator, NodeLookup { return false; } + @Override + public boolean isReady() { + return true; + } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 0a59392789a..c4ff23a2c7f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -116,8 +116,11 @@ public interface ModelContext { default List<TenantSecretStore> tenantSecretStores() { return List.of(); } - /// Default setting for the gc-options attribute if not specified explicit by application - String jvmGCOptions(); + // Default setting for the gc-options attribute if not specified explicit by application + default String jvmGCOptions() { return jvmGCOptions(Optional.empty()); } + + // Default setting for the gc-options attribute if not specified explicit by application + String jvmGCOptions(Optional<ClusterSpec.Type> clusterType); // Note: Used in unit tests (set to false in TestProperties) to avoid needing to deal with implicitly created node for logserver default boolean useDedicatedNodeForLogserver() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 306f36e7674..64fc6d1dc16 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -71,7 +71,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean hostedVespa() { return hostedVespa; } @Override public Zone zone() { return zone; } @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } - @Override public String jvmGCOptions() { return jvmGCOptions; } + @Override public String jvmGCOptions(Optional<ClusterSpec.Type> clusterType) { return jvmGCOptions; } @Override public String feedSequencerType() { return sequencerType; } @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return false; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 3e421f9ba05..4a415fccbcc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -6,6 +6,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Dictionary; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Ranking; import com.yahoo.searchdefinition.document.Sorting; @@ -251,9 +252,25 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce ib.hnsw.multithreadedindexing(params.multiThreadedIndexing()); aaB.index(ib); } + Dictionary dictionary = attribute.getDictionary(); + if (dictionary != null) { + aaB.dictionary.type(convert(dictionary.getType())); + } return aaB; } + private static AttributesConfig.Attribute.Dictionary.Type.Enum convert(Dictionary.Type type) { + switch (type) { + case BTREE: + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + case HASH: + return AttributesConfig.Attribute.Dictionary.Type.HASH; + case BTREE_AND_HASH: + return AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH; + } + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + } + public void getConfig(AttributesConfig.Builder builder, FieldSet fs) { for (Attribute attribute : attributes.values()) { if (isAttributeInFieldSet(attribute, fs)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 8cf862a72af..f230a7c10eb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -78,6 +78,8 @@ public final class Attribute implements Cloneable, Serializable { /** The aliases for this attribute */ private final Set<String> aliases = new LinkedHashSet<>(); + private Dictionary dictionary = new Dictionary(); + /** * True if this attribute should be returned during first pass of search. * Null means make the default decision for this kind of attribute @@ -208,6 +210,7 @@ public final class Attribute implements Cloneable, Serializable { public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } public Sorting getSorting() { return sorting; } + public Dictionary getDictionary() { return dictionary; } public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; } public void setCreateIfNonExistent(boolean create) { this.createIfNonExistent = create; } @@ -231,6 +234,7 @@ public final class Attribute implements Cloneable, Serializable { public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } public void setDistanceMetric(DistanceMetric metric) { this.distanceMetric = Optional.of(metric); } public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } + public void setDictionary(Dictionary dictionary) { this.dictionary = dictionary; } public String getName() { return name; } public Type getType() { return type; } @@ -348,7 +352,7 @@ public final class Attribute implements Cloneable, Serializable { @Override public int hashCode() { return Objects.hash( - name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, + name, type, collectionType, sorting, dictionary, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, distanceMetric, hnswIndexParams); } @@ -370,10 +374,10 @@ public final class Attribute implements Cloneable, Serializable { if (this.createIfNonExistent != other.createIfNonExistent) return false; if (this.enableBitVectors != other.enableBitVectors) return false; if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false; - // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now if (this.fastSearch != other.fastSearch) return false; if (this.huge != other.huge) return false; if (! this.sorting.equals(other.sorting)) return false; + if (! Objects.equals(dictionary, other.dictionary)) return false; if (! Objects.equals(tensorType, other.tensorType)) return false; if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java new file mode 100644 index 00000000000..e492d572f27 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java @@ -0,0 +1,16 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.document; + +/** + * Represents settings for dictionary control + * + * @author baldersheim + */ +public class Dictionary { + public enum Type { BTREE, HASH, BTREE_AND_HASH }; + private final Type type; + public Dictionary() { this(Type.BTREE); } + public Dictionary(Type type) { this.type = type; } + public Type getType() { return type; } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index d81394382c2..76b707fa19b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -80,6 +80,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, */ private Matching matching = new Matching(); + private Dictionary dictionary = null; + /** Attribute settings, or null if there are none */ private final Map<String, Attribute> attributes = new TreeMap<>(); @@ -533,6 +535,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, public void setMatching(Matching matching) { this.matching=matching; } /** + * Returns Dictionary settings. + */ + public Dictionary getDictionary() { return dictionary; } + + + public void setDictionary(Dictionary dictionary) { this.dictionary=dictionary; } + + /** * Set the matching type for this field and all subfields. */ // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java index b638932a4a8..56e241adb8e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java @@ -90,10 +90,6 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain this.enableOnlyBitVector = enableOnlyBitVector; } - public boolean isDoAlias() { - return doAlias; - } - public void setDoAlias(boolean doAlias) { this.doAlias = doAlias; } @@ -106,9 +102,6 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain this.alias = alias; } - public String getAliasedName() { - return aliasedName; - } public void setAliasedName(String aliasedName) { this.aliasedName = aliasedName; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java new file mode 100644 index 00000000000..ce7c5a71a21 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.SDField; + +/** + * Represents operations controlling setup of dictionary used for queries + * + * @author baldersheim + */ +public class DictionaryOperation implements FieldOperation { + private final Dictionary.Type type; + + public DictionaryOperation(Dictionary.Type type) { + this.type = type; + } + @Override + public void apply(SDField field) { + Dictionary prev = field.getDictionary(); + if (prev == null) { + field.setDictionary(new Dictionary(type)); + } else if ((prev.getType() == Dictionary.Type.BTREE && type == Dictionary.Type.HASH) || + (prev.getType() == Dictionary.Type.HASH && type == Dictionary.Type.BTREE)) + { + field.setDictionary(new Dictionary(Dictionary.Type.BTREE_AND_HASH)); + } else { + if (prev.getType() != type) { + throw new IllegalArgumentException("Can not combine previous dictionary setting " + prev.getType() + + " with current " + type); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java new file mode 100644 index 00000000000..fd567ec2d54 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.NumericDataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Propagates dictionary settings from field level to attribute level. + * Only applies to numeric fields with fast-search enabled. + * + * @author baldersheim + */ +public class DictionaryProcessor extends Processor { + public DictionaryProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : search.allConcreteFields()) { + Dictionary dictionary = field.getDictionary(); + if (dictionary == null) continue; + + Attribute attribute = field.getAttribute(); + if (attribute.getDataType().getPrimitiveType() instanceof NumericDataType ) { + if (attribute.isFastSearch()) { + attribute.setDictionary(dictionary); + } else { + fail(search, field, "You must specify 'attribute:fast-search' to allow dictionary control"); + } + } else { + fail(search, field, "You can only specify 'dictionary:' for numeric fields"); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 79f19efe422..ff0edcd0404 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -15,7 +15,13 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.ArrayList; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 1a3ef9e54b4..136d352ece7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -37,6 +37,7 @@ public class Processing { AttributesImplicitWord::new, MutableAttributes::new, CreatePositionZCurve::new, + DictionaryProcessor::new, WordMatch::new, ImportedFieldsResolver::new, ImplicitSummaries::new, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index 3744af7cc2c..61b5e6f2a64 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -25,7 +25,7 @@ import java.util.logging.Level; public abstract class Processor { protected final Search search; - protected DeployLogger deployLogger; + protected final DeployLogger deployLogger; protected final RankProfileRegistry rankProfileRegistry; protected final QueryProfiles queryProfiles; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index e5d8f9add3f..43b394b7f42 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -3,11 +3,14 @@ package com.yahoo.vespa.model.admin; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import java.util.Optional; + /** * @author hmusum */ @@ -18,7 +21,7 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain addDefaultHandlersWithVip(); addLogHandler(); - setJvmGCOptions(deployState.getProperties().jvmGCOptions()); + setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index dd16f50b392..c4bd7198e11 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -5,9 +5,12 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; +import java.util.Optional; + /** * Container cluster for cluster-controller containers. * @@ -25,7 +28,7 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC addDefaultHandlersWithVip(); this.featureFlags = deployState.featureFlags(); this.reindexingContext = createReindexingContext(deployState); - setJvmGCOptions(deployState.getProperties().jvmGCOptions()); + setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 45af078f2a3..89182ca28ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -46,6 +46,7 @@ public class MetricsProxyContainer extends Container implements private final Optional<ClusterMembership> clusterMembership; private final ModelContext.FeatureFlags featureFlags; private final MetricsProxyContainerCluster cluster; + private final String jvmGCOptions; public MetricsProxyContainer(MetricsProxyContainerCluster cluster, HostResource host, int index, DeployState deployState) { @@ -54,6 +55,7 @@ public class MetricsProxyContainer extends Container implements this.clusterMembership = host.spec().membership(); this.featureFlags = deployState.featureFlags(); this.cluster = cluster; + this.jvmGCOptions = deployState.getProperties().jvmGCOptions(clusterMembership.map(membership -> membership.cluster().type())); setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); addNodeSpecificComponents(); @@ -151,6 +153,7 @@ public class MetricsProxyContainer extends Container implements int maxHeapSize = featureFlags.metricsProxyMaxHeapSizeInMb(clusterMembership.get().cluster().type()); builder.jvm .verbosegc(true) + .gcopts(jvmGCOptions) .heapsize(maxHeapSize); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 194967f9868..e504bee0a30 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -96,7 +96,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addPlatformBundle(METRICS_PROXY_BUNDLE_FILE); addClusterComponents(); - setJvmGCOptions(deployState.getProperties().jvmGCOptions()); } private void addClusterComponents() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 9f98fdb4ea2..06e02821544 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -4,10 +4,13 @@ package com.yahoo.vespa.model.container.http.ssl; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * Component specification for {@link com.yahoo.jdisc.http.server.jetty.ConnectorFactory} with hosted specific configuration. @@ -76,6 +79,11 @@ public class HostedSslConnectorFactory extends ConnectorFactory { // Disables TLSv1.3 as it causes some browsers to prompt user for client certificate (when connector has 'want' auth) connectorBuilder.ssl.enabledProtocols(List.of("TLSv1.2")); + // Add TLS_RSA_WITH_AES_256_GCM_SHA384 cipher to list of defalt allowed ciphers + Set<String> ciphers = new HashSet<>(TlsContext.ALLOWED_CIPHER_SUITES); + ciphers.add("TLS_RSA_WITH_AES_256_GCM_SHA384"); + connectorBuilder.ssl.enabledCipherSuites(Set.copyOf(ciphers)); + connectorBuilder .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(true)) .idleTimeout(Duration.ofMinutes(3).toSeconds()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 7b3bc498164..f2e8757c115 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -273,6 +273,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addCloudSecretStore(ApplicationContainerCluster cluster, Element secretStoreElement, DeployState deployState) { + if ( ! deployState.isHosted()) return; CloudSecretStore cloudSecretStore = new CloudSecretStore(); Map<String, TenantSecretStore> secretStoresByName = deployState.getProperties().tenantSecretStores() .stream() @@ -621,6 +622,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ? (deployState.isHosted() ? ContainerCluster.CMS : ContainerCluster.G1GC) : options; } + private static String getJvmOptions(ApplicationContainerCluster cluster, Element nodesElement, DeployLogger deployLogger) { String jvmOptions; if (nodesElement.hasAttribute(VespaDomBuilder.JVM_OPTIONS)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java index 909a29fd0a3..dcb9ca17ef4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java @@ -97,10 +97,10 @@ public class IndexedHierarchicDistributionValidator { } if (totalReadyCopies % groupCount != 0) { throw new IllegalArgumentException(getErrorMsgPrefix(clusterName) + "Expected equal amount of ready copies per group, but " + - totalReadyCopies + " ready copies is specified with " + groupCount + " groups"); + totalReadyCopies + " ready copies is specified with " + groupCount + " groups"); } if (totalReadyCopies == 0) { - System.err.println(getErrorMsgPrefix(clusterName) + "Warning. No ready copies configured. At least one is recommended."); + throw new IllegalArgumentException(getErrorMsgPrefix(clusterName) + "Warning. No ready copies configured. At least one is required."); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 869f671d8ef..f8a648006e9 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -252,6 +252,7 @@ TOKEN : | < PROPERTIES: "properties" > | < ATTRIBUTE: "attribute" > | < SORTING: "sorting" > +| < DICTIONARY: "dictionary" > | < ASCENDING: "ascending" > | < DESCENDING: "descending" > | < UCA: "uca" > @@ -267,6 +268,8 @@ TOKEN : | < IDENTICAL: "identical" > | < STEMMING: "stemming" > | < NORMALIZING: "normalizing" > +| < HASH: "hash" > +| < BTREE: "btree" > | < BOLDING: "bolding" > | < BODY: "body" > | < HEADER: "header" > @@ -987,6 +990,7 @@ String fieldBody(SDField field, Search search, SDDocumentType document) : { } attribute(field) | body(field) | bolding(field) | + dictionary(field) | fieldStemming(field) | header(field) | id(field, document) | @@ -1520,6 +1524,34 @@ void bolding(FieldOperationContainer field) : } /** + * This rule consumes a dictionary statement of a field element. + * + * @param field The field to modify. + */ +void dictionary(FieldOperationContainer field) : +{ + Dictionary.Type type; +} +{ + <DICTIONARY> <COLON> type = dictionaryType() + { + field.addOperation(new DictionaryOperation(type)); + } +} + +Dictionary.Type dictionaryType() : +{ + Dictionary.Type type; +} +{ + ( <HASH> { type = Dictionary.Type.HASH; } + | <BTREE> { type = Dictionary.Type.BTREE; } ) + { + return type; + } +} + +/** * This rule consumes a body statement of a field element. * * @param field The field to modify. @@ -2609,6 +2641,7 @@ String identifier() : { } | <ATTRIBUTE> | <BODY> | <BOLDING> + | <BTREE> | <COMPRESSION> | <COMPRESSIONLEVEL> | <COMPRESSIONTHRESHOLD> @@ -2616,6 +2649,7 @@ String identifier() : { } | <CREATEIFNONEXISTENT> | <DENSEPOSTINGLISTTHRESHOLD> | <DESCENDING> + | <DICTIONARY> | <DIRECT> | <DOCUMENT> | <DOCUMENTSUMMARY> @@ -2637,6 +2671,7 @@ String identifier() : { } | <FULL> | <FUNCTION> | <GRAM> + | <HASH> | <HEADER> | <HUGE> | <ID> diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 0d4b3bc4ca7..641e2d8fde5 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "location_zcurve" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index dc269284e8f..bb4ba665406 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "elem_array.name" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -30,6 +31,7 @@ attribute[].name "elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index c43e38d0561..d05d2d1d5e0 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "singlebyte" attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "multibyte" attribute[].datatype INT8 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "wsbyte" attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "singleint" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "multiint" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "wsint" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "singlelong" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "multilong" attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "wslong" attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "singlefloat" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "multifloat" attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -310,6 +321,7 @@ attribute[].name "wsfloat" attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -338,6 +350,7 @@ attribute[].name "singledouble" attribute[].datatype DOUBLE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -366,6 +379,7 @@ attribute[].name "multidouble" attribute[].datatype DOUBLE attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -394,6 +408,7 @@ attribute[].name "wsdouble" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -422,6 +437,7 @@ attribute[].name "singlestring" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -450,6 +466,7 @@ attribute[].name "multistring" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -478,6 +495,7 @@ attribute[].name "wsstring" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index 4ecb0c2af8f..e9f3f68adc1 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "a1" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "a2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "a3" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "a5" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "a6" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "b1" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "b2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "b3" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "b4" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "b5" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "b6" attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -310,6 +321,7 @@ attribute[].name "b7" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -338,6 +350,7 @@ attribute[].name "a9" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -366,6 +379,7 @@ attribute[].name "a10" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -394,6 +408,7 @@ attribute[].name "a11" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -422,6 +437,7 @@ attribute[].name "a12" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -450,6 +466,7 @@ attribute[].name "a7_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -478,6 +495,7 @@ attribute[].name "a8_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index fe6f42e55bc..622fb9f349c 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "prefixenabled" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "fleeting" attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "fleeting2" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "foundat" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "collapseby" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "ts" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "combineda" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "year_arr" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "year_sub" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg index 53eb98b6ba8..4d275787bfd 100644 --- a/config-model/src/test/derived/hnsw_index/attributes.cfg +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "t1" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "t2" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg index 2c209db96f2..bfdf90ac12c 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "ref_from_a" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "ref_from_b" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "from_a_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "from_b_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index 5bb0d9bfef3..f1b20c6e454 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "parent_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "my_pos_zcurve" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index f299756efb1..9e0b5f18170 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "parent_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "my_elem_array.name" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -58,6 +60,7 @@ attribute[].name "my_elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "my_elem_map.key" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -114,6 +118,7 @@ attribute[].name "my_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -142,6 +147,7 @@ attribute[].name "my_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "my_str_int_map.key" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -198,6 +205,7 @@ attribute[].name "my_str_int_map.value" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 680d6571939..68cd917336f 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "a_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "b_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "b_ref_with_summary" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "my_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "my_string_field" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "my_int_array_field" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "my_int_wset_field" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "my_ancient_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 52367a04a30..a931af7af4d 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "onlygrandparent" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "overridden" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "onlymother" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index b25a46dc433..11498de54b1 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index d9624e89500..acbdf119d0d 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "str_map.key" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -30,6 +31,7 @@ attribute[].name "str_map.value" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "int_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 5fd10269ceb..ecc8c2fd69d 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "str_elem_map.key" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -30,6 +31,7 @@ attribute[].name "str_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "str_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "int_elem_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "int_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index ebb8eb10c14..a31325e67d0 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "sales" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "hiphopvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "metalvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index 5f5e7e62d83..68f24871f02 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "sales" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "scorekey" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 6a7ff8af7ad..ff45cf1a41e 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "some_predicate_field" attribute[].datatype PREDICATE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index 2a5fc890f9e..f9878068372 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "attributefield1" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "attributefield2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index 3ecd24d50dc..ad3ff5e62f8 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "campaign_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "other_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "yet_another_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index a84c7780965..ebe0e83540e 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "syntaxcheck" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "syntaxcheck2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "infieldonly" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index cc28f1b6f84..398cdf5f8f8 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "f2" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "f3" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "f4" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "f5" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "f6" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index d30c79efbc2..05e19a15d96 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "abyte" attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "along" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "abool" attribute[].datatype BOOL attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "ashortfloat" attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "arrayfield" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "setfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "setfield2" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "setfield3" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "setfield4" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent true attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "tagfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "juletre" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -310,6 +321,7 @@ attribute[].name "album1" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -338,6 +350,7 @@ attribute[].name "other" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index 911f8e797e1..5589ad018a7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.SearchBuilder; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java new file mode 100644 index 00000000000..ba51caca0f7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java @@ -0,0 +1,143 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.test.TestUtil; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.search.AttributesConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Test configuration of dictionary control. + * + * @author baldersheim + */ +public class DictionaryTestCase { + private static AttributesConfig getConfig(Search search) { + AttributeFields attributes = new AttributeFields(search); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder); + return builder.build(); + } + private Search createSearch(String def) throws ParseException { + SearchBuilder sb = SearchBuilder.createFromString(def); + return sb.getSearch(); + } + @Test + public void testDefaultDictionarySettings() throws ParseException { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field s1 type string {", + " indexing: attribute | summary", + " }", + " field n1 type int {", + " indexing: summary | attribute", + " }", + " }", + "}"); + Search search = createSearch(def); + assertEquals(Dictionary.Type.BTREE, search.getAttribute("s1").getDictionary().getType()); + assertEquals(Dictionary.Type.BTREE, search.getAttribute("n1").getDictionary().getType()); + } + + void verifyNumericDictionaryControl(Dictionary.Type expected, + AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig, + String type, + String ... cfg) throws ParseException + { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field n1 type " + type + " {", + " indexing: summary | attribute", + " attribute:fast-search", + TestUtil.joinLines(cfg), + " }", + " }", + "}"); + Search search = createSearch(def); + assertEquals(expected, search.getAttribute("n1").getDictionary().getType()); + assertEquals(expectedConfig, + getConfig(search).attribute().get(0).dictionary().type()); + } + + @Test + public void testNumericBtreeSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.BTREE, + AttributesConfig.Attribute.Dictionary.Type.BTREE, + "int", + "dictionary:btree"); + } + @Test + public void testNumericHashSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.HASH, + AttributesConfig.Attribute.Dictionary.Type.HASH, + "int", + "dictionary:hash"); + } + @Test + public void testNumericBtreeAndHashSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.BTREE_AND_HASH, + AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, + "int", + "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNumericArrayBtreeAndHashSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.BTREE_AND_HASH, + AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, + "array<int>", + "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNumericWSetBtreeAndHashSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.BTREE_AND_HASH, + AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, + "weightedset<int>", + "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNonNumericFieldsFailsDictionaryControl() throws ParseException { + String def = + "search test {\n" + + " document test {\n" + + " field n1 type string {\n" + + " indexing: summary | attribute\n" + + " dictionary:btree\n" + + " }\n" + + " }\n" + + "}\n"; + try { + SearchBuilder sb = SearchBuilder.createFromString(def); + fail("Controlling dictionary for non-numeric fields are not yet supported."); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': You can only specify 'dictionary:' for numeric fields", e.getMessage()); + } + } + @Test + public void testNonFastSearchFieldsFailsDictionaryControl() throws ParseException { + String def = + "search test {\n" + + " document test {\n" + + " field n1 type int {\n" + + " indexing: summary | attribute\n" + + " dictionary:btree\n" + + " }\n" + + " }\n" + + "}\n"; + try { + SearchBuilder sb = SearchBuilder.createFromString(def); + fail("Controlling dictionary for non-fast-search fields are not allowed."); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index e43b5085528..7082720f721 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -40,6 +40,7 @@ import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.prelude.cluster.QrMonitorConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; @@ -53,6 +54,7 @@ import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.hamcrest.Matchers; +import org.hamcrest.core.IsEqual; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -61,6 +63,8 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -79,6 +83,7 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; @@ -87,6 +92,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -726,7 +732,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " </secret-store>", "</container>"); try { - createModel(root, clusterElem); + DeployState state = new DeployState.Builder() + .properties(new TestProperties().setHostedVespa(true)) + .zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName())) + .build(); + createModel(root, state, null, clusterElem); fail("secret store not defined"); } catch (RuntimeException e) { assertEquals("No configured secret store named store1", e.getMessage()); @@ -928,6 +938,30 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void require_allowed_ciphers() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + nodesXml, + "</container>" ); + + DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))).build(); + createModel(root, state, null, clusterElem); + ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); + + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); + ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == 4443).findFirst().orElseThrow(); + ConnectorConfig.Builder builder = new ConnectorConfig.Builder(); + tlsPort.getConfig(builder); + + ConnectorConfig connectorConfig = new ConnectorConfig(builder); + Set<String> expectedCiphers = new HashSet<>(); + expectedCiphers.add("TLS_RSA_WITH_AES_256_GCM_SHA384"); + expectedCiphers.addAll(TlsContext.ALLOWED_CIPHER_SUITES); + + assertThat(connectorConfig.ssl().enabledCipherSuites(), containsInAnyOrder(expectedCiphers.toArray())); + } + + @Test public void cluster_with_zookeeper() { Function<Integer, String> servicesXml = (nodeCount) -> "<container version='1.0' id='default'>" + "<nodes count='" + nodeCount + "'/>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java index 80ab6745b79..3be592e54e7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -254,10 +254,4 @@ public class IndexedHierarchicDistributionTest { getTwoGroupsCluster(4, 2, "2|*"); } - @Test - public void allowNoReadyCopies() throws Exception { - // The active one should be indexed anyhow. Setting up no ready copies - getTwoGroupsCluster(4, 0, "2|*"); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 7f563b876a7..0548bc7520f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -316,4 +316,6 @@ public class NodeResources { return value; } + public static NodeResources zero() { return new NodeResources(0, 0, 0, 0); } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java index 08f9b81fec7..45d6bfbe370 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java @@ -10,14 +10,26 @@ import com.yahoo.config.provision.SystemName; * @author hakonhall */ public interface ZoneApi { + SystemName getSystemName(); ZoneId getId(); + + /** + * Returns the virtual ID of this zone. For ordinary zones this is the same as {@link ZoneApi#getId()}, for a + * system represented as a zone this is a fixed ID that is independent of the actual zone ID. + */ + default ZoneId getVirtualId() { + return getId(); + } + default Environment getEnvironment() { return getId().environment(); } + default RegionName getRegionName() { return getId().region(); } CloudName getCloudName(); /** Returns the region name within the cloud, e.g. 'us-east-1' in AWS */ String getCloudNativeRegionName(); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java index f79abbddc56..d3f80d8c3f5 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java @@ -1,7 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index be336c22cb6..939a0b8476a 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -4,7 +4,9 @@ namespace=vespa.config.search attribute[].name string attribute[].datatype enum { STRING, BOOL, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE +# Deprecated/ do-not-use, will soon be GCed. attribute[].dictionary.ordering enum { ORDERED, UNORDERED } default = ORDERED +attribute[].dictionary.type enum { BTREE, HASH, BTREE_AND_HASH } default = BTREE attribute[].removeifzero bool default=false attribute[].createifnonexistent bool default=false attribute[].fastsearch bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 5d46f3dc240..4d7f1349e59 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -74,7 +74,6 @@ import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantMetaData; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.stats.LockStats; import com.yahoo.vespa.curator.stats.ThreadLockStats; @@ -505,7 +504,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye NestedTransaction transaction = new NestedTransaction(); Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId)) .map(lock -> new ApplicationTransaction(lock, transaction)); - try (var sessionLock = tenantApplications.lock(applicationId)) { + try (var applicationLock = tenantApplications.lock(applicationId)) { Optional<Long> activeSession = tenantApplications.activeSessionOf(applicationId); if (activeSession.isEmpty()) return false; @@ -517,6 +516,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } Curator curator = tenantRepository.getCurator(); + CompletionWaiter waiter = tenantApplications.createRemoveApplicationWaiter(applicationId); transaction.add(new ContainerEndpointsCache(tenant.getPath(), curator).delete(applicationId)); // TODO: Not unit tested // Delete any application roles transaction.add(new ApplicationRolesStore(curator, tenant.getPath()).delete(applicationId)); @@ -533,6 +533,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } else { transaction.commit(); } + + // Wait for app being removed on other servers + waiter.awaitCompletion(Duration.ofSeconds(30)); + return true; } finally { applicationTransaction.ifPresent(ApplicationTransaction::close); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 7fcda86aa98..523334dda7f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import ai.vespa.util.http.VespaAsyncHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClient.java index 1ced0c4ce4f..1887bf1db9d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClient.java @@ -1,7 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import ai.vespa.util.http.VespaAsyncHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.concurrent.CompletableFutures; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 6f626464c63..405ddee17e8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -43,6 +43,8 @@ import java.util.concurrent.ExecutorService; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.config.server.tenant.TenantRepository.getBarriersPath; +import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static java.util.stream.Collectors.toSet; /** @@ -55,6 +57,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica private static final Logger log = Logger.getLogger(TenantApplications.class.getName()); + private final Curator curator; private final ApplicationCuratorDatabase database; private final Curator.DirectoryCache directoryCache; private final Executor zkWatcherExecutor; @@ -67,11 +70,13 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica private final MetricUpdater tenantMetricUpdater; private final Clock clock; private final TenantFileSystemDirs tenantFileSystemDirs; + private final ConfigserverConfig configserverConfig; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, ExecutorService zkCacheExecutor, Metrics metrics, ReloadListener reloadListener, ConfigserverConfig configserverConfig, HostRegistry hostRegistry, TenantFileSystemDirs tenantFileSystemDirs, Clock clock) { + this.curator = curator; this.database = new ApplicationCuratorDatabase(tenant, curator); this.tenant = tenant; this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command); @@ -85,6 +90,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica this.hostRegistry = hostRegistry; this.tenantFileSystemDirs = tenantFileSystemDirs; this.clock = clock; + this.configserverConfig = configserverConfig; } // For testing only @@ -252,28 +258,36 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica } } + // Note: Assumes that caller already holds the application lock + // (when getting event from zookeeper to remove application, + // the lock should be held by the thread that causes the event to happen) public void removeApplication(ApplicationId applicationId) { - try (Lock lock = lock(applicationId)) { - if (exists(applicationId)) { - log.log(Level.INFO, "Tried removing application " + applicationId + ", but it seems to have been deployed again"); - return; - } + if (exists(applicationId)) { + log.log(Level.INFO, "Tried removing application " + applicationId + ", but it seems to have been deployed again"); + return; + } - if (applicationMapper.hasApplication(applicationId, clock.instant())) { - applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); - reloadListenersOnRemove(applicationId); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); - log.log(Level.INFO, "Application removed: " + applicationId); - } + if (hasApplication(applicationId)) { + applicationMapper.remove(applicationId); + hostRegistry.removeHostsForKey(applicationId); + reloadListenersOnRemove(applicationId); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + getRemoveApplicationWaiter(applicationId).notifyCompletion(); + log.log(Level.INFO, "Application removed: " + applicationId); } } + public boolean hasApplication(ApplicationId applicationId) { + return applicationMapper.hasApplication(applicationId, clock.instant()); + } + public void removeApplicationsExcept(Set<ApplicationId> applications) { for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { if ( ! applications.contains(activeApplication)) { - removeApplication(activeApplication); + try (var applicationLock = lock(activeApplication)){ + removeApplication(activeApplication); + } } } } @@ -405,4 +419,27 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } + public CompletionWaiter createRemoveApplicationWaiter(ApplicationId applicationId) { + Path barrierPath = getRemoveApplicationBarrierPath(); + curator.create(barrierPath); + return curator.createCompletionWaiter(barrierPath, + removeApplicationWaiterNode(applicationId), + curator.zooKeeperEnsembleCount(), + configserverConfig.serverId()); + } + + public CompletionWaiter getRemoveApplicationWaiter(ApplicationId applicationId) { + return curator.getCompletionWaiter(getRemoveApplicationBarrierPath().append(removeApplicationWaiterNode(applicationId)), + curator.zooKeeperEnsembleCount(), + configserverConfig.serverId()); + } + + private Path getRemoveApplicationBarrierPath() { + return getBarriersPath().append(tenant.value()); + } + + private String removeApplicationWaiterNode(ApplicationId applicationId) { + return applicationId.serializedForm() + "-delete"; + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 5f89903bf07..490e82ff10a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -32,13 +32,17 @@ import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.flags.UnboundFlag; import java.io.File; import java.net.URI; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.ToIntFunction; import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; @@ -260,8 +264,7 @@ public class ModelContextImpl implements ModelContext { private final Quota quota; private final List<TenantSecretStore> tenantSecretStores; private final SecretStore secretStore; - - private final String jvmGcOptions; + private final StringFlag jvmGCOptionsFlag; public Properties(ApplicationId applicationId, ConfigserverConfig configserverConfig, @@ -294,8 +297,8 @@ public class ModelContextImpl implements ModelContext { this.quota = maybeQuota.orElseGet(Quota::unlimited); this.tenantSecretStores = tenantSecretStores; this.secretStore = secretStore; - - jvmGcOptions = flagValue(flagSource, applicationId, PermanentFlags.JVM_GC_OPTIONS); + this.jvmGCOptionsFlag = PermanentFlags.JVM_GC_OPTIONS.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()); } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -355,13 +358,12 @@ public class ModelContextImpl implements ModelContext { return SecretStoreExternalIdRetriever.populateExternalId(secretStore, applicationId.tenant(), zone.system(), tenantSecretStores); } - @Override public String jvmGCOptions() { return jvmGcOptions; } - - private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { - return flag.bindTo(source) - .with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()) - .boxedValue(); + @Override public String jvmGCOptions(Optional<ClusterSpec.Type> clusterType) { + return clusterType.map(type -> jvmGCOptionsFlag.with(CLUSTER_TYPE, type.name())) + .orElse(jvmGCOptionsFlag) + .value(); } + } private static NodeResources parseDedicatedClusterControllerFlavor(String flagValue) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index e859de1c599..ee041ed3490 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.yolean.Exceptions; import org.apache.http.client.HttpClient; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java index 71eed450955..796c581b3c2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java @@ -1,17 +1,15 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever; import com.yahoo.yolean.Exceptions; import org.apache.http.client.methods.HttpPost; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 9c77442dcc8..350a693a82c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; import java.util.logging.Level; import org.apache.http.HttpEntity; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 163d26036a9..26765615233 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.yolean.Exceptions; import org.apache.http.client.HttpClient; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java index 1e523b37917..067be8102b8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java index d6feecb7479..b92f8fda88f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java @@ -1,7 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index b4d3f305f61..1734b84ec43 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -77,6 +77,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.curator.Curator.CompletionWaiter; + /** * * Session repository for a tenant. Stores session state in zookeeper and file system. There are two @@ -225,7 +227,7 @@ public class SessionRepository { logger.log(Level.FINE, "Created application " + params.getApplicationId()); long sessionId = session.getSessionId(); SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); - Curator.CompletionWaiter waiter = sessionZooKeeperClient.createPrepareWaiter(); + CompletionWaiter waiter = sessionZooKeeperClient.createPrepareWaiter(); Optional<ApplicationSet> activeApplicationSet = getActiveApplicationSet(params.getApplicationId()); ConfigChangeActions actions = sessionPreparer.prepare(applicationRepo.getHostValidator(), logger, params, activeApplicationSet, now, getSessionAppDir(sessionId), @@ -409,11 +411,11 @@ public class SessionRepository { void activate(RemoteSession session) { long sessionId = session.getSessionId(); - Curator.CompletionWaiter waiter = createSessionZooKeeperClient(sessionId).getActiveWaiter(); + CompletionWaiter waiter = createSessionZooKeeperClient(sessionId).getActiveWaiter(); log.log(Level.FINE, () -> session.logPre() + "Activating " + sessionId); applicationRepo.activateApplication(ensureApplicationLoaded(session), sessionId); log.log(Level.FINE, () -> session.logPre() + "Notifying " + waiter); - notifyCompletion(waiter, session); + notifyCompletion(waiter); log.log(Level.INFO, session.logPre() + "Session activated: " + sessionId); } @@ -431,9 +433,9 @@ public class SessionRepository { void prepareRemoteSession(RemoteSession session) { SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); - Curator.CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); + CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); ensureApplicationLoaded(session); - notifyCompletion(waiter, session); + notifyCompletion(waiter); } public ApplicationSet ensureApplicationLoaded(RemoteSession session) { @@ -453,14 +455,14 @@ public class SessionRepository { } void confirmUpload(Session session) { - Curator.CompletionWaiter waiter = session.getSessionZooKeeperClient().getUploadWaiter(); + CompletionWaiter waiter = session.getSessionZooKeeperClient().getUploadWaiter(); long sessionId = session.getSessionId(); log.log(Level.FINE, "Notifying upload waiter for session " + sessionId); - notifyCompletion(waiter, session); + notifyCompletion(waiter); log.log(Level.FINE, "Done notifying upload for session " + sessionId); } - void notifyCompletion(Curator.CompletionWaiter completionWaiter, Session session) { + void notifyCompletion(CompletionWaiter completionWaiter) { try { completionWaiter.notifyCompletion(); } catch (RuntimeException e) { @@ -474,8 +476,7 @@ public class SessionRepository { KeeperException.NodeExistsException.class); Class<? extends Throwable> exceptionClass = e.getCause().getClass(); if (acceptedExceptions.contains(exceptionClass)) - log.log(Level.FINE, "Not able to notify completion for session " + session.getSessionId() + - " (" + completionWaiter + ")," + + log.log(Level.FINE, "Not able to notify completion for session (" + completionWaiter + ")," + " node " + (exceptionClass.equals(KeeperException.NoNodeException.class) ? "has been deleted" : "already exists")); @@ -618,7 +619,7 @@ public class SessionRepository { log.log(Level.FINE, () -> TenantRepository.logPre(tenantName) + "Creating session " + sessionId + " in ZooKeeper"); SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); sessionZKClient.createNewSession(clock.instant()); - Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); + CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); LocalSession session = new LocalSession(tenantName, sessionId, app, sessionZKClient); waiter.awaitCompletion(Duration.ofSeconds(Math.min(60, timeoutBudget.timeLeft().getSeconds()))); addLocalSession(session); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 92d7f60d69b..c7c4f1926d7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; @@ -34,6 +34,7 @@ import java.util.List; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static com.yahoo.yolean.Exceptions.uncheck; /** @@ -95,23 +96,15 @@ public class SessionZooKeeperClient { } } - Curator.CompletionWaiter createPrepareWaiter() { - return createCompletionWaiter(PREPARE_BARRIER); - } + public CompletionWaiter createActiveWaiter() { return createCompletionWaiter(ACTIVE_BARRIER); } - public Curator.CompletionWaiter createActiveWaiter() { - return createCompletionWaiter(ACTIVE_BARRIER); - } + CompletionWaiter createPrepareWaiter() { return createCompletionWaiter(PREPARE_BARRIER); } - Curator.CompletionWaiter getPrepareWaiter() { - return getCompletionWaiter(getWaiterPath(PREPARE_BARRIER)); - } + CompletionWaiter getPrepareWaiter() { return getCompletionWaiter(getWaiterPath(PREPARE_BARRIER)); } - Curator.CompletionWaiter getActiveWaiter() { - return getCompletionWaiter(getWaiterPath(ACTIVE_BARRIER)); - } + CompletionWaiter getActiveWaiter() { return getCompletionWaiter(getWaiterPath(ACTIVE_BARRIER)); } - Curator.CompletionWaiter getUploadWaiter() { return getCompletionWaiter(getWaiterPath(UPLOAD_BARRIER)); } + CompletionWaiter getUploadWaiter() { return getCompletionWaiter(getWaiterPath(UPLOAD_BARRIER)); } private static final String PREPARE_BARRIER = "prepareBarrier"; private static final String ACTIVE_BARRIER = "activeBarrier"; @@ -126,11 +119,11 @@ public class SessionZooKeeperClient { return (curator.zooKeeperEnsembleCount() / 2) + 1; // majority } - private Curator.CompletionWaiter createCompletionWaiter(String waiterNode) { + private CompletionWaiter createCompletionWaiter(String waiterNode) { return curator.createCompletionWaiter(sessionPath, waiterNode, getNumberOfMembers(), serverId); } - private Curator.CompletionWaiter getCompletionWaiter(Path path) { + private CompletionWaiter getCompletionWaiter(Path path) { return curator.getCompletionWaiter(path, getNumberOfMembers(), serverId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index babb7e4a596..0c2d69ad882 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -89,6 +89,7 @@ public class TenantRepository { private static final Path tenantsPath = Path.fromString("/config/v2/tenants/"); private static final Path locksPath = Path.fromString("/config/v2/locks/"); + private static final Path barriersPath = Path.fromString("/config/v2/barriers/"); private static final Path vespaPath = Path.fromString("/vespa"); private static final Duration checkForRemovedApplicationsInterval = Duration.ofMinutes(1); private static final Logger log = Logger.getLogger(TenantRepository.class.getName()); @@ -199,6 +200,7 @@ public class TenantRepository { curator.create(tenantsPath); curator.create(locksPath); + curator.create(barriersPath); createSystemTenants(configserverConfig); curator.create(vespaPath); @@ -596,6 +598,10 @@ public class TenantRepository { return locksPath.append(tenantName.value()); } + public static Path getBarriersPath() { + return barriersPath; + } + public Curator getCurator() { return curator; } } diff --git a/container-core/pom.xml b/container-core/pom.xml index 138a68a7c0f..ea6e7d32310 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -150,6 +150,16 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + <dependency> <!-- TODO Vespa 8: stop providing org.json:json --> <groupId>org.json</groupId> <artifactId>json</artifactId> diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java index a4a092b02ad..b7dd3a5e239 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; @@ -11,7 +11,6 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.BasicResponseHandler; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import java.io.IOException; import java.net.URI; diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java index 5e3ec8f2f05..6270d1c7b06 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java @@ -1,7 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; diff --git a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonMapper.java b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonMapper.java new file mode 100644 index 00000000000..5a5a990737c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonMapper.java @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +/** + * Default Jackson {@link ObjectMapper} instance shared by {@link com.yahoo.restapi}. + * + * @author bjorncs + */ +class JacksonJsonMapper { + + static final ObjectMapper instance = new ObjectMapper() + .registerModule(new JavaTimeModule()) + .registerModule(new Jdk8Module()) + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + + private JacksonJsonMapper() {} +} diff --git a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java index 0a2c08530aa..1b356b3b459 100644 --- a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java @@ -2,13 +2,12 @@ package com.yahoo.restapi; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.databind.ObjectWriter; import com.yahoo.container.jdisc.HttpResponse; -import java.util.logging.Level; import java.io.IOException; import java.io.OutputStream; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -19,30 +18,39 @@ import java.util.logging.Logger; public class JacksonJsonResponse<T> extends HttpResponse { private static final Logger log = Logger.getLogger(JacksonJsonResponse.class.getName()); - private static final ObjectMapper defaultJsonMapper = - new ObjectMapper().registerModule(new JavaTimeModule()).registerModule(new Jdk8Module()); private final ObjectMapper jsonMapper; + private final boolean prettyPrint; private final T entity; public JacksonJsonResponse(int statusCode, T entity) { - this(statusCode, entity, defaultJsonMapper); + this(statusCode, entity, false); + } + + public JacksonJsonResponse(int statusCode, T entity, boolean prettyPrint) { + this(statusCode, entity, JacksonJsonMapper.instance, prettyPrint); } public JacksonJsonResponse(int statusCode, T entity, ObjectMapper jsonMapper) { + this(statusCode, entity, jsonMapper, false); + } + + public JacksonJsonResponse(int statusCode, T entity, ObjectMapper jsonMapper, boolean prettyPrint) { super(statusCode); this.entity = entity; this.jsonMapper = jsonMapper; + this.prettyPrint = prettyPrint; } @Override public void render(OutputStream outputStream) throws IOException { + ObjectWriter writer = prettyPrint ? jsonMapper.writerWithDefaultPrettyPrinter() : jsonMapper.writer(); if (log.isLoggable(Level.FINE)) { - String json = jsonMapper.writeValueAsString(entity); + String json = writer.writeValueAsString(entity); log.log(Level.FINE, "Writing the following JSON to response output stream:\n" + json); outputStream.write(json.getBytes()); } else { - jsonMapper.writeValue(outputStream, entity); + writer.writeValue(outputStream, entity); } } diff --git a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java index 32ea3ae708f..43ca0dab29e 100644 --- a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java @@ -14,6 +14,10 @@ public class MessageResponse extends SlimeJsonResponse { super(slime(message)); } + public MessageResponse(int statusCode, String message) { + super(statusCode, slime(message)); + } + private static Slime slime(String message) { var slime = new Slime(); slime.setObject().setString("message", message); diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApi.java b/container-core/src/main/java/com/yahoo/restapi/RestApi.java new file mode 100644 index 00000000000..08bac710001 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/RestApi.java @@ -0,0 +1,115 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Slime; + +import java.io.InputStream; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalLong; + +/** + * Rest API routing and response serialization + * + * @author bjorncs + */ +public interface RestApi { + + static Builder builder() { return new RestApiImpl.BuilderImpl(); } + static RouteBuilder route(String pathPattern) { return new RestApiImpl.RouteBuilderImpl(pathPattern); } + + HttpResponse handleRequest(HttpRequest request); + ObjectMapper jacksonJsonMapper(); + + interface Builder { + Builder setObjectMapper(ObjectMapper mapper); + Builder setDefaultRoute(RouteBuilder route); + Builder addRoute(RouteBuilder route); + Builder addFilter(Filter filter); + <EXCEPTION extends RuntimeException> Builder addExceptionMapper(Class<EXCEPTION> type, ExceptionMapper<EXCEPTION> mapper); + <ENTITY> Builder addResponseMapper(Class<ENTITY> type, ResponseMapper<ENTITY> mapper); + Builder disableDefaultExceptionMappers(); + Builder disableDefaultResponseMappers(); + RestApi build(); + } + + interface RouteBuilder { + RouteBuilder name(String name); + RouteBuilder get(MethodHandler<?> handler); + RouteBuilder post(MethodHandler<?> handler); + RouteBuilder put(MethodHandler<?> handler); + RouteBuilder delete(MethodHandler<?> handler); + RouteBuilder patch(MethodHandler<?> handler); + RouteBuilder defaultHandler(MethodHandler<?> handler); + RouteBuilder addFilter(Filter filter); + } + + @FunctionalInterface interface ExceptionMapper<EXCEPTION extends RuntimeException> { HttpResponse toResponse(EXCEPTION exception, RequestContext context); } + + @FunctionalInterface interface MethodHandler<ENTITY> { ENTITY handleRequest(RequestContext context) throws RestApiException; } + + @FunctionalInterface interface ResponseMapper<ENTITY> { HttpResponse toHttpResponse(ENTITY responseEntity, RequestContext context); } + + @FunctionalInterface interface Filter { HttpResponse filterRequest(FilterContext context); } + + /** Marker interface required for automatic serialization of Jackson response entities */ + interface JacksonResponseEntity {} + + /** Marker interface required for automatic serialization of Jackson request entities */ + interface JacksonRequestEntity {} + + interface RequestContext { + HttpRequest request(); + PathParameters pathParameters(); + QueryParameters queryParameters(); + Headers headers(); + Attributes attributes(); + Optional<RequestContent> requestContent(); + RequestContent requestContentOrThrow(); + + interface Parameters { + Optional<String> getString(String name); + String getStringOrThrow(String name); + default Optional<Boolean> getBoolean(String name) { return getString(name).map(Boolean::valueOf);} + default boolean getBooleanOrThrow(String name) { return Boolean.parseBoolean(getStringOrThrow(name)); } + default OptionalLong getLong(String name) { + return getString(name).map(Long::parseLong).map(OptionalLong::of).orElseGet(OptionalLong::empty); + } + default long getLongOrThrow(String name) { return Long.parseLong(getStringOrThrow(name)); } + default OptionalDouble getDouble(String name) { + return getString(name).map(Double::parseDouble).map(OptionalDouble::of).orElseGet(OptionalDouble::empty); + } + default double getDoubleOrThrow(String name) { return Double.parseDouble(getStringOrThrow(name)); } + } + + interface PathParameters extends Parameters {} + interface QueryParameters extends Parameters {} + interface Headers extends Parameters {} + + interface Attributes { + Optional<Object> get(String name); + void set(String name, Object value); + } + + interface RequestContent { + String contentType(); + InputStream inputStream(); + ObjectMapper jacksonJsonMapper(); + byte[] consumeByteArray(); + String consumeString(); + JsonNode consumeJsonNode(); + Slime consumeSlime(); + <T extends JacksonRequestEntity> T consumeJacksonEntity(Class<T> type); + } + } + + interface FilterContext { + RequestContext requestContext(); + String route(); + HttpResponse executeNext(); + } +} diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiException.java b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java new file mode 100644 index 00000000000..ac3aa647b87 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; + +import java.util.function.Function; + +/** + * A {@link RuntimeException} that represents a http response. + * + * @author bjorncs + */ +public class RestApiException extends RuntimeException { + private final int statusCode; + private final HttpResponse response; + + public RestApiException(int statusCode, String errorType, String message) { + this(new ErrorResponse(statusCode, errorType, message), message, null); + } + + public RestApiException(HttpResponse response, String message) { + this(response, message, null); + } + + public RestApiException(int statusCode, String errorType, String message, Throwable cause) { + this(new ErrorResponse(statusCode, errorType, message), message, cause); + } + + public RestApiException(HttpResponse response, String message, Throwable cause) { + super(message, cause); + this.statusCode = response.getStatus(); + this.response = response; + } + + private RestApiException(Function<String, HttpResponse> responseFromMessage, String message, Throwable cause) { + this(responseFromMessage.apply(message), message, cause); + } + + public int statusCode() { return statusCode; } + public HttpResponse response() { return response; } + + public static class NotFoundException extends RestApiException { + public NotFoundException() { super(ErrorResponse::notFoundError, "Not Found", null); } + } + + public static class MethodNotAllowed extends RestApiException { + public MethodNotAllowed() { super(ErrorResponse::methodNotAllowed, "Method not allowed", null); } + public MethodNotAllowed(HttpRequest request) { + super(ErrorResponse::methodNotAllowed, "Method '" + request.getMethod().name() + "' is not allowed", null); + } + } + + public static class BadRequest extends RestApiException { + public BadRequest(String message) { super(ErrorResponse::badRequest, message, null); } + public BadRequest(String message, Throwable cause) { super(ErrorResponse::badRequest, message, cause); } + } + + public static class InternalServerError extends RestApiException { + public InternalServerError(String message) { super(ErrorResponse::internalServerError, message, null); } + public InternalServerError(String message, Throwable cause) { super(ErrorResponse::internalServerError, message, cause); } + } + + public static class Forbidden extends RestApiException { + public Forbidden(String message) { super(ErrorResponse::forbidden, message, null); } + public Forbidden(String message, Throwable cause) { super(ErrorResponse::forbidden, message, cause); } + } +} diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java new file mode 100644 index 00000000000..af816f41411 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -0,0 +1,399 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +class RestApiImpl implements RestApi { + + private static final Logger log = Logger.getLogger(RestApiImpl.class.getName()); + + private final Route defaultRoute; + private final List<Route> routes; + private final List<ExceptionMapperHolder<?>> exceptionMappers; + private final List<ResponseMapperHolder<?>> responseMappers; + private final List<Filter> filters; + private final ObjectMapper jacksonJsonMapper; + + private RestApiImpl(RestApi.Builder builder) { + BuilderImpl builderImpl = (BuilderImpl) builder; + ObjectMapper jacksonJsonMapper = builderImpl.jacksonJsonMapper != null ? builderImpl.jacksonJsonMapper : JacksonJsonMapper.instance.copy(); + this.defaultRoute = builderImpl.defaultRoute != null ? builderImpl.defaultRoute : createDefaultRoute(); + this.routes = List.copyOf(builderImpl.routes); + this.exceptionMappers = combineWithDefaultExceptionMappers( + builderImpl.exceptionMappers, Boolean.TRUE.equals(builderImpl.disableDefaultExceptionMappers)); + this.responseMappers = combineWithDefaultResponseMappers( + builderImpl.responseMappers, jacksonJsonMapper, Boolean.TRUE.equals(builderImpl.disableDefaultResponseMappers)); + this.filters = List.copyOf(builderImpl.filters); + this.jacksonJsonMapper = jacksonJsonMapper; + } + + @Override + public HttpResponse handleRequest(HttpRequest request) { + Path pathMatcher = new Path(request.getUri()); + Route resolvedRoute = resolveRoute(pathMatcher); + RequestContextImpl requestContext = new RequestContextImpl(request, pathMatcher, jacksonJsonMapper); + FilterContextImpl filterContext = + createFilterContextRecursive( + resolvedRoute, requestContext, filters, + createFilterContextRecursive(resolvedRoute, requestContext, resolvedRoute.filters, null)); + if (filterContext != null) { + return filterContext.executeFirst(); + } else { + return dispatchToRoute(resolvedRoute, requestContext); + } + } + + @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } + + private HttpResponse dispatchToRoute(Route route, RequestContextImpl context) { + RestApi.MethodHandler<?> resolvedHandler = route.handlerPerMethod.get(context.request().getMethod()); + if (resolvedHandler == null) { + resolvedHandler = route.defaultHandler; + } + Object entity; + try { + entity = resolvedHandler.handleRequest(context); + } catch (RuntimeException e) { + ExceptionMapperHolder<?> mapper = exceptionMappers.stream() + .filter(holder -> holder.matches(e)) + .findFirst().orElseThrow(() -> e); + return mapper.toResponse(e, context); + } + if (entity == null) throw new NullPointerException("Handler must return non-null value"); + ResponseMapperHolder<?> mapper = responseMappers.stream() + .filter(holder -> holder.matches(entity)) + .findFirst().orElseThrow(() -> new IllegalStateException("No mapper configured for " + entity.getClass())); + return mapper.toHttpResponse(entity, context); + } + + private Route resolveRoute(Path pathMatcher) { + Route matchingRoute = routes.stream() + .filter(route -> pathMatcher.matches(route.pathPattern)) + .findFirst() + .orElse(null); + if (matchingRoute != null) return matchingRoute; + pathMatcher.matches(defaultRoute.pathPattern); // to populate any path parameters + return defaultRoute; + } + + private FilterContextImpl createFilterContextRecursive( + Route route, RequestContextImpl requestContext, List<Filter> filters, FilterContextImpl previousContext) { + FilterContextImpl filterContext = previousContext; + ListIterator<Filter> iterator = filters.listIterator(filters.size()); + while (iterator.hasPrevious()) { + filterContext = new FilterContextImpl(route, iterator.previous(), requestContext, filterContext); + } + return filterContext; + } + + private static Route createDefaultRoute() { + RouteBuilder routeBuilder = new RouteBuilderImpl("{*}") + .defaultHandler(context -> { + throw new RestApiException.NotFoundException(); + }); + return ((RouteBuilderImpl)routeBuilder).build(); + } + + private static List<ExceptionMapperHolder<?>> combineWithDefaultExceptionMappers( + List<ExceptionMapperHolder<?>> configuredExceptionMappers, boolean disableDefaultMappers) { + List<ExceptionMapperHolder<?>> exceptionMappers = new ArrayList<>(configuredExceptionMappers); + if (!disableDefaultMappers){ + exceptionMappers.add(new ExceptionMapperHolder<>(RestApiException.class, (exception, context) -> exception.response())); + } + return exceptionMappers; + } + + private static List<ResponseMapperHolder<?>> combineWithDefaultResponseMappers( + List<ResponseMapperHolder<?>> configuredResponseMappers, ObjectMapper jacksonJsonMapper, boolean disableDefaultMappers) { + List<ResponseMapperHolder<?>> responseMappers = new ArrayList<>(configuredResponseMappers); + if (!disableDefaultMappers) { + responseMappers.add(new ResponseMapperHolder<>(HttpResponse.class, (entity, context) -> entity)); + responseMappers.add(new ResponseMapperHolder<>(String.class, (entity, context) -> new MessageResponse(entity))); + responseMappers.add(new ResponseMapperHolder<>(Slime.class, (entity, context) -> new SlimeJsonResponse(entity))); + responseMappers.add(new ResponseMapperHolder<>(JsonNode.class, (entity, context) -> new JacksonJsonResponse<>(200, entity, jacksonJsonMapper, true))); + responseMappers.add(new ResponseMapperHolder<>(RestApi.JacksonResponseEntity.class, (entity, context) -> new JacksonJsonResponse<>(200, entity, jacksonJsonMapper, true))); + } + return responseMappers; + } + + static class BuilderImpl implements RestApi.Builder { + private final List<Route> routes = new ArrayList<>(); + private final List<ExceptionMapperHolder<?>> exceptionMappers = new ArrayList<>(); + private final List<ResponseMapperHolder<?>> responseMappers = new ArrayList<>(); + private final List<RestApi.Filter> filters = new ArrayList<>(); + private Route defaultRoute; + private ObjectMapper jacksonJsonMapper; + private Boolean disableDefaultExceptionMappers; + private Boolean disableDefaultResponseMappers; + + @Override public RestApi.Builder setObjectMapper(ObjectMapper mapper) { this.jacksonJsonMapper = mapper; return this; } + @Override public RestApi.Builder setDefaultRoute(RestApi.RouteBuilder route) { this.defaultRoute = ((RouteBuilderImpl)route).build(); return this; } + @Override public RestApi.Builder addRoute(RestApi.RouteBuilder route) { routes.add(((RouteBuilderImpl)route).build()); return this; } + @Override public RestApi.Builder addFilter(RestApi.Filter filter) { filters.add(filter); return this; } + + @Override public <EXCEPTION extends RuntimeException> RestApi.Builder addExceptionMapper(Class<EXCEPTION> type, RestApi.ExceptionMapper<EXCEPTION> mapper) { + exceptionMappers.add(new ExceptionMapperHolder<>(type, mapper)); return this; + } + + @Override public <ENTITY> RestApi.Builder addResponseMapper(Class<ENTITY> type, RestApi.ResponseMapper<ENTITY> mapper) { + responseMappers.add(new ResponseMapperHolder<>(type, mapper)); return this; + } + + @Override public Builder disableDefaultExceptionMappers() { this.disableDefaultExceptionMappers = true; return this; } + @Override public Builder disableDefaultResponseMappers() { this.disableDefaultResponseMappers = true; return this; } + @Override public RestApi build() { return new RestApiImpl(this); } + } + + public static class RouteBuilderImpl implements RestApi.RouteBuilder { + private final String pathPattern; + private String name; + private final Map<com.yahoo.jdisc.http.HttpRequest.Method, RestApi.MethodHandler<?>> handlerPerMethod = new HashMap<>(); + private final List<RestApi.Filter> filters = new ArrayList<>(); + private RestApi.MethodHandler<?> defaultHandler; + + RouteBuilderImpl(String pathPattern) { this.pathPattern = pathPattern; } + + @Override public RestApi.RouteBuilder name(String name) { this.name = name; return this; } + @Override public RestApi.RouteBuilder get(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.GET, handler); } + @Override public RestApi.RouteBuilder post(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.POST, handler); } + @Override public RestApi.RouteBuilder put(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.PUT, handler); } + @Override public RestApi.RouteBuilder delete(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.DELETE, handler); } + @Override public RestApi.RouteBuilder patch(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.PATCH, handler); } + @Override public RestApi.RouteBuilder defaultHandler(RestApi.MethodHandler<?> handler) { defaultHandler = handler; return this; } + @Override public RestApi.RouteBuilder addFilter(RestApi.Filter filter) { filters.add(filter); return this; } + + private RestApi.RouteBuilder addHandler(com.yahoo.jdisc.http.HttpRequest.Method method, RestApi.MethodHandler<?> handler) { + handlerPerMethod.put(method, handler); return this; + } + + private Route build() { return new Route(this); } + } + + private static class RequestContextImpl implements RestApi.RequestContext { + final HttpRequest request; + final Path pathMatcher; + final ObjectMapper jacksonJsonMapper; + final PathParameters pathParameters = new PathParametersImpl(); + final QueryParameters queryParameters = new QueryParametersImpl(); + final Headers headers = new HeadersImpl(); + final Attributes attributes = new AttributesImpl(); + final RequestContent requestContent; + + RequestContextImpl(HttpRequest request, Path pathMatcher, ObjectMapper jacksonJsonMapper) { + this.request = request; + this.pathMatcher = pathMatcher; + this.jacksonJsonMapper = jacksonJsonMapper; + this.requestContent = request.getData() != null ? new RequestContentImpl() : null; + } + + @Override public HttpRequest request() { return request; } + @Override public PathParameters pathParameters() { return pathParameters; } + @Override public QueryParameters queryParameters() { return queryParameters; } + @Override public Headers headers() { return headers; } + @Override public Attributes attributes() { return attributes; } + @Override public Optional<RequestContent> requestContent() { return Optional.ofNullable(requestContent); } + @Override public RequestContent requestContentOrThrow() { + return requestContent().orElseThrow(() -> new RestApiException.BadRequest("Request content missing")); + } + + private class PathParametersImpl implements RestApi.RequestContext.PathParameters { + @Override + public Optional<String> getString(String name) { + if (name.equals("*")) { + String rest = pathMatcher.getRest(); + return rest.isEmpty() ? Optional.empty() : Optional.of(rest); + } + return Optional.ofNullable(pathMatcher.get(name)); + } + @Override public String getStringOrThrow(String name) { + return getString(name) + .orElseThrow(() -> new RestApiException.BadRequest("Path parameter '" + name + "' is missing")); + } + } + + private class QueryParametersImpl implements RestApi.RequestContext.QueryParameters { + @Override public Optional<String> getString(String name) { return Optional.ofNullable(request.getProperty(name)); } + @Override public String getStringOrThrow(String name) { + return getString(name) + .orElseThrow(() -> new RestApiException.BadRequest("Query parameter '" + name + "' is missing")); + } + } + + private class HeadersImpl implements RestApi.RequestContext.Headers { + @Override public Optional<String> getString(String name) { return Optional.ofNullable(request.getHeader(name)); } + @Override public String getStringOrThrow(String name) { + return getString(name) + .orElseThrow(() -> new RestApiException.BadRequest("Header '" + name + "' missing")); + } + } + + private class RequestContentImpl implements RestApi.RequestContext.RequestContent { + @Override public String contentType() { return request.getHeader("Content-Type"); } + @Override public InputStream inputStream() { return request.getData(); } + @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } + @Override public byte[] consumeByteArray() { return convertIoException(() -> inputStream().readAllBytes()); } + @Override public String consumeString() { return new String(consumeByteArray(), StandardCharsets.UTF_8); } + + @Override + public JsonNode consumeJsonNode() { + return convertIoException(() -> { + try { + if (log.isLoggable(Level.FINE)) { + String content = consumeString(); + log.fine(() -> "Request content: " + content); + return jacksonJsonMapper.readTree(content); + } else { + return jacksonJsonMapper.readTree(request.getData()); + } + } catch (com.fasterxml.jackson.core.JsonParseException e) { + log.log(Level.FINE, e.getMessage(), e); + throw new RestApiException.BadRequest("Invalid json request content: " + Exceptions.toMessageString(e), e); + } + }); + } + + @Override + public Slime consumeSlime() { + try { + String content = consumeString(); + log.fine(() -> "Request content: " + content); + return SlimeUtils.jsonToSlimeOrThrow(content); + } catch (com.yahoo.slime.JsonParseException e) { + log.log(Level.FINE, e.getMessage(), e); + throw new RestApiException.BadRequest("Invalid json request content: " + Exceptions.toMessageString(e), e); + } + } + + @Override + public <T extends JacksonRequestEntity> T consumeJacksonEntity(Class<T> type) { + return convertIoException(() -> { + try { + if (log.isLoggable(Level.FINE)) { + String content = consumeString(); + log.fine(() -> "Request content: " + content); + return jacksonJsonMapper.readValue(content, type); + } else { + return jacksonJsonMapper.readValue(request.getData(), type); + } + } catch (com.fasterxml.jackson.core.JsonParseException | JsonMappingException e) { + log.log(Level.FINE, e.getMessage(), e); + throw new RestApiException.BadRequest("Invalid json request content: " + Exceptions.toMessageString(e), e); + } + }); + } + } + + private class AttributesImpl implements RestApi.RequestContext.Attributes { + @Override public Optional<Object> get(String name) { return Optional.ofNullable(request.getJDiscRequest().context().get(name)); } + @Override public void set(String name, Object value) { request.getJDiscRequest().context().put(name, value); } + } + + @FunctionalInterface private interface SupplierThrowingIoException<T> { T get() throws IOException; } + private static <T> T convertIoException(SupplierThrowingIoException<T> supplier) { + try { + return supplier.get(); + } catch (IOException e) { + throw new RestApiException.InternalServerError("Failed to read request content: " + Exceptions.toMessageString(e), e); + } + } + } + + private class FilterContextImpl implements RestApi.FilterContext { + final Route route; + final RestApi.Filter filter; + final RequestContextImpl requestContext; + final FilterContextImpl next; + + FilterContextImpl(Route route, RestApi.Filter filter, RequestContextImpl requestContext, FilterContextImpl next) { + this.route = route; + this.filter = filter; + this.requestContext = requestContext; + this.next = next; + } + + @Override public RestApi.RequestContext requestContext() { return requestContext; } + @Override public String route() { return route.name != null ? route.name : route.pathPattern; } + + HttpResponse executeFirst() { return filter.filterRequest(this); } + + @Override + public HttpResponse executeNext() { + if (next != null) { + return next.filter.filterRequest(next); + } else { + return dispatchToRoute(route, requestContext); + } + } + } + + private static class ExceptionMapperHolder<EXCEPTION extends RuntimeException> { + final Class<EXCEPTION> type; + final RestApi.ExceptionMapper<EXCEPTION> mapper; + + ExceptionMapperHolder(Class<EXCEPTION> type, RestApi.ExceptionMapper<EXCEPTION> mapper) { + this.type = type; + this.mapper = mapper; + } + + boolean matches(RuntimeException e) { return type.isAssignableFrom(e.getClass()); } + HttpResponse toResponse(RuntimeException e, RestApi.RequestContext context) { return mapper.toResponse(type.cast(e), context); } + } + + private static class ResponseMapperHolder<ENTITY> { + final Class<ENTITY> type; + final RestApi.ResponseMapper<ENTITY> mapper; + + ResponseMapperHolder(Class<ENTITY> type, RestApi.ResponseMapper<ENTITY> mapper) { + this.type = type; + this.mapper = mapper; + } + + boolean matches(Object entity) { return type.isAssignableFrom(entity.getClass()); } + HttpResponse toHttpResponse(Object entity, RestApi.RequestContext context) { return mapper.toHttpResponse(type.cast(entity), context); } + } + + + static class Route { + private final String pathPattern; + private final String name; + private final Map<com.yahoo.jdisc.http.HttpRequest.Method, RestApi.MethodHandler<?>> handlerPerMethod; + private final RestApi.MethodHandler<?> defaultHandler; + private final List<Filter> filters; + + private Route(RestApi.RouteBuilder builder) { + RouteBuilderImpl builderImpl = (RouteBuilderImpl)builder; + this.pathPattern = builderImpl.pathPattern; + this.name = builderImpl.name; + this.handlerPerMethod = Map.copyOf(builderImpl.handlerPerMethod); + this.defaultHandler = builderImpl.defaultHandler != null ? builderImpl.defaultHandler : createDefaultMethodHandler(); + this.filters = List.copyOf(builderImpl.filters); + } + + private RestApi.MethodHandler<?> createDefaultMethodHandler() { + return context -> { throw new RestApiException.MethodNotAllowed(context.request()); }; + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java new file mode 100644 index 00000000000..9fe813903dd --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; + +/** + * @author bjorncs + */ +public abstract class RestApiRequestHandler<T extends RestApiRequestHandler<T>> extends LoggingRequestHandler { + + private final RestApi restApi; + + @FunctionalInterface public interface RestApiProvider<T> { RestApi createRestApi(T self); } + + /** + * RestApi will usually refer to handler methods of subclass, which are not accessible before super constructor has completed. + * This is hack to leak reference to subclass instance's "this" reference. + * Caller must ensure that provider instance does not try to access any uninitialized fields. + */ + @SuppressWarnings("unchecked") + protected RestApiRequestHandler(LoggingRequestHandler.Context context, RestApiProvider<T> provider) { + super(context); + this.restApi = provider.createRestApi((T)this); + } + + protected RestApiRequestHandler(LoggingRequestHandler.Context context, RestApi restApi) { + super(context); + this.restApi = restApi; + } + + @Override public final HttpResponse handle(HttpRequest request) { return restApi.handleRequest(request); } + + protected RestApi restApi() { return restApi; } +} diff --git a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java new file mode 100644 index 00000000000..16cc2353986 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java @@ -0,0 +1,125 @@ +package com.yahoo.restapi;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.yolean.Exceptions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.yahoo.jdisc.http.HttpRequest.Method; +import static com.yahoo.restapi.RestApi.route; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bjorncs + */ +class RestApiImplTest { + + @Test + void routes_requests_to_correct_handler() { + RestApi restApi = RestApi.builder() + .addRoute(route("/api1/{*}").get(ctx -> new MessageResponse("get-method-response"))) + .addRoute(route("/api2/{*}").post(ctx -> new MessageResponse("post-method-response"))) + .setDefaultRoute(route("{*}").defaultHandler(ctx -> ErrorResponse.notFoundError("default-method-response"))) + .build(); + verifyJsonResponse(restApi, Method.GET, "/api1/subpath", null, 200, "{\"message\":\"get-method-response\"}"); + verifyJsonResponse(restApi, Method.POST, "/api1/subpath", "{}", 405, null); + verifyJsonResponse(restApi, Method.GET, "/api2/subpath", null, 405, null); + verifyJsonResponse(restApi, Method.POST, "/api2/subpath", "{}", 200, "{\"message\":\"post-method-response\"}"); + verifyJsonResponse(restApi, Method.PUT, "/api2/subpath", "{}", 405, null); + verifyJsonResponse(restApi, Method.GET, "/unknown/subpath", null, 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"default-method-response\"}"); + verifyJsonResponse(restApi, Method.DELETE, "/unknown/subpath", "{}", 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"default-method-response\"}"); + } + + @Test + void executes_filters_and_handler_in_correct_order() { + List<String> actualEvaluationOrdering = new ArrayList<>(); + RestApi.MethodHandler<HttpResponse> handler = context -> { + actualEvaluationOrdering.add("handler"); + return new MessageResponse("get-method-response"); + }; + class NamedTestFilter implements RestApi.Filter { + final String name; + NamedTestFilter(String name) { this.name = name; } + + @Override + public HttpResponse filterRequest(RestApi.FilterContext context) { + actualEvaluationOrdering.add("pre-" + name); + HttpResponse response = context.executeNext(); + actualEvaluationOrdering.add("post-" + name); + return response; + } + } + RestApi restApi = RestApi.builder() + .setDefaultRoute(route("{*}") + .defaultHandler(handler) + .addFilter(new NamedTestFilter("route-filter-1")) + .addFilter(new NamedTestFilter("route-filter-2"))) + .addFilter(new NamedTestFilter("global-filter-1")) + .addFilter(new NamedTestFilter("global-filter-2")) + .build(); + verifyJsonResponse(restApi, Method.GET, "/", null, 200, "{\"message\":\"get-method-response\"}"); + List<String> expectedOrdering = List.of( + "pre-global-filter-1", "pre-global-filter-2", "pre-route-filter-1", "pre-route-filter-2", + "handler", + "post-route-filter-2", "post-route-filter-1", "post-global-filter-2", "post-global-filter-1"); + assertEquals(expectedOrdering, actualEvaluationOrdering); + } + + @SuppressWarnings("divzero") + @Test + void handles_custom_response_and_exception_mapper() { + RestApi restApi = RestApi.builder() + .disableDefaultExceptionMappers() + .disableDefaultResponseMappers() + .addRoute(route("/long").get(ctx -> 123456L)) + .addRoute(route("/exception").get(ctx -> 123L / 0L)) + .addResponseMapper(Long.class, (entity, ctx) -> new MessageResponse("long value is " + entity)) + .addExceptionMapper(ArithmeticException.class, (exception, ctx) -> ErrorResponse.internalServerError("oops division by zero")) + .build(); + verifyJsonResponse(restApi, Method.GET, "/long", null, 200, "{\"message\":\"long value is 123456\"}"); + verifyJsonResponse(restApi, Method.GET, "/exception", null, 500, "{\"message\":\"oops division by zero\", \"error-code\":\"INTERNAL_SERVER_ERROR\"}"); + } + + @Test + void method_handler_can_consume_and_produce_json() { + RestApi restApi = RestApi.builder() + .addRoute(route("/api").post( + ctx -> ctx.requestContent().get().consumeJacksonEntity(TestEntity.class))) + .build(); + String rawJson = "{\"mystring\":\"my-string-value\", \"myinstant\":\"2000-01-01T00:00:00Z\"}"; + verifyJsonResponse(restApi, Method.POST, "/api", rawJson, 200, rawJson); + } + + private static void verifyJsonResponse(RestApi restApi, Method method, String path, String requestContent, int expectedStatusCode, String expectedJson) { + HttpRequest testRequest = requestContent != null ? + HttpRequest.createTestRequest( + path, method, + new ByteArrayInputStream(requestContent.getBytes(StandardCharsets.UTF_8)), + Map.of("Content-Type", "application/json")) : + HttpRequest.createTestRequest(path, method); + HttpResponse response = restApi.handleRequest(testRequest); + assertEquals(expectedStatusCode, response.getStatus()); + if (expectedJson != null) { + assertEquals("application/json", response.getContentType()); + var outputStream = new ByteArrayOutputStream(); + Exceptions.uncheck(() -> response.render(outputStream)); + String content = outputStream.toString(StandardCharsets.UTF_8); + JsonTestHelper.assertJsonEquals(content, expectedJson); + } + } + + public static class TestEntity implements RestApi.JacksonRequestEntity, RestApi.JacksonResponseEntity { + @JsonProperty("mystring") public String stringValue; + @JsonProperty("myinstant") public Instant instantValue; + } +}
\ No newline at end of file diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index a278421f603..2c5d764c013 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -461,7 +461,7 @@ <jetty.version>9.4.38.v20210224</jetty.version> <org.lz4.version>1.7.1</org.lz4.version> <org.json.version>20090211</org.json.version> - <slf4j.version>1.7.5</slf4j.version> + <slf4j.version>1.7.30</slf4j.version> <xml-apis.version>1.4.01</xml-apis.version> <!-- These must be kept in sync with version used by current jersey2.version. --> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java index 719f948eaa9..85b042d584c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopRoleService.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.aws; import com.yahoo.config.provision.TenantName; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -18,6 +17,11 @@ public class NoopRoleService implements RoleService { } @Override + public TenantRoles getTenantRole(TenantName tenant) { + return new TenantRoles(tenant.value() + "-host-role", tenant.value() + "-tenant-role"); + } + + @Override public void deleteTenantRole(TenantName tenant) { } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java index ac499a0def3..d27fa0a5bd8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/RoleService.java @@ -13,6 +13,9 @@ public interface RoleService { Optional<TenantRoles> createTenantRole(TenantName tenant); + /** Retrieve the names of the tenant roles (host and container). Does not guarantee these roles exist */ + TenantRoles getTenantRole(TenantName tenant); + void deleteTenantRole(TenantName tenant); String createTenantPolicy(TenantName tenant, String policyName, String awsId, String role); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index b9a81ba8a02..8c1fc57f4b6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -53,6 +53,7 @@ public class Node { private final String flavor; private final String clusterId; private final ClusterType clusterType; + private final boolean retired; private final boolean wantToRetire; private final boolean wantToDeprovision; private final Optional<TenantName> reservedTo; @@ -67,7 +68,7 @@ public class Node { Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, - int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision, + int cost, String flavor, String clusterId, ClusterType clusterType, boolean retired, boolean wantToRetire, boolean wantToDeprovision, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports, List<NodeHistory> history, Set<String> additionalIpAddresses, String openStackId, Optional<String> switchHostname) { @@ -93,6 +94,7 @@ public class Node { this.flavor = flavor; this.clusterId = clusterId; this.clusterType = clusterType; + this.retired = retired; this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; this.reservedTo = reservedTo; @@ -200,6 +202,10 @@ public class Node { return clusterType; } + public boolean retired() { + return retired; + } + public boolean wantToRetire() { return wantToRetire; } @@ -302,6 +308,7 @@ public class Node { private String flavor; private String clusterId; private ClusterType clusterType; + private boolean retired; private boolean wantToRetire; private boolean wantToDeprovision; private Optional<TenantName> reservedTo = Optional.empty(); @@ -339,6 +346,7 @@ public class Node { this.flavor = node.flavor; this.clusterId = node.clusterId; this.clusterType = node.clusterType; + this.retired = node.retired; this.wantToRetire = node.wantToRetire; this.wantToDeprovision = node.wantToDeprovision; this.reservedTo = node.reservedTo; @@ -470,6 +478,11 @@ public class Node { return this; } + public Builder retired(boolean retired) { + this.retired = retired; + return this; + } + public Builder wantToRetire(boolean wantToRetire) { this.wantToRetire = wantToRetire; return this; @@ -514,7 +527,7 @@ public class Node { return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, - cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo, exclusiveTo, + cost, flavor, clusterId, clusterType, retired, wantToRetire, wantToDeprovision, reservedTo, exclusiveTo, wantedDockerImage, currentDockerImage, reports, history, additionalIpAddresses, openStackId, switchHostname); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index c3cb904f545..bbe982bd5fe 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -143,6 +143,7 @@ public interface NodeRepository { node.getFlavor(), clusterIdOf(node.getMembership()), clusterTypeOf(node.getMembership()), + Optional.ofNullable(node.getMembership()).map(NodeMembership::getRetired).orElse(false), node.getWantToRetire(), node.getWantToDeprovision(), Optional.ofNullable(node.getReservedTo()).map(TenantName::from), @@ -180,6 +181,8 @@ public interface NodeRepository { case proxyhost: return NodeType.proxyhost; case config: return NodeType.config; case confighost: return NodeType.confighost; + case controller: return NodeType.controller; + case controllerhost: return NodeType.controllerhost; default: throw new IllegalArgumentException("Unknown type: " + nodeType); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index b9ee696431b..76637d10a6e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -37,15 +38,9 @@ public interface ZoneRegistry { /** Returns the default region for the given environment, if one is configured */ Optional<RegionName> getDefaultRegion(Environment environment); - /** Returns the API endpoints of all known config servers in the given zone */ - List<URI> getConfigServerUris(ZoneId zoneId); - /** Returns the URI for the config server VIP in the given zone */ URI getConfigServerVipUri(ZoneId zoneId); - /** Returns all possible API endpoints of all known config servers and config server VIPs in the given zone */ - List<URI> getConfigServerApiUris(ZoneId zoneId); - /** Returns the time to live for deployments in the given zone, or empty if this is infinite */ Optional<Duration> getDeploymentTimeToLive(ZoneId zoneId); @@ -55,6 +50,9 @@ public interface ZoneRegistry { /** Returns the system of this registry */ SystemName system(); + /** Returns the system of this registry as a zone */ + ZoneApi systemZone(); + /** Return the configserver's Athenz service identity */ AthenzIdentity getConfigServerHttpsIdentity(ZoneId zoneId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 51b7a24b5e4..4351ac17001 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -329,18 +329,24 @@ public class ApplicationController { }); } - public LockedApplication withNewInstance(LockedApplication application, ApplicationId id) { - if (id.instance().isTester()) - throw new IllegalArgumentException("'" + id + "' is a tester application!"); - InstanceId.validate(id.instance().value()); + /** Fetches the requested application package from the artifact store(s). */ + public ApplicationPackage getApplicationPackage(ApplicationId id, ApplicationVersion version) { + return new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)); + } - if (getInstance(id).isPresent()) - throw new IllegalArgumentException("Could not create '" + id + "': Instance already exists"); - if (getInstance(dashToUnderscore(id)).isPresent()) // VESPA-1945 - throw new IllegalArgumentException("Could not create '" + id + "': Instance " + dashToUnderscore(id) + " already exists"); + /** Returns given application with a new instance */ + public LockedApplication withNewInstance(LockedApplication application, ApplicationId instance) { + if (instance.instance().isTester()) + throw new IllegalArgumentException("'" + instance + "' is a tester application!"); + InstanceId.validate(instance.instance().value()); - log.info("Created " + id); - return application.withNewInstance(id.instance()); + if (getInstance(instance).isPresent()) + throw new IllegalArgumentException("Could not create '" + instance + "': Instance already exists"); + if (getInstance(dashToUnderscore(instance)).isPresent()) // VESPA-1945 + throw new IllegalArgumentException("Could not create '" + instance + "': Instance " + dashToUnderscore(instance) + " already exists"); + + log.info("Created " + instance); + return application.withNewInstance(instance.instance()); } /** Deploys an application package for an existing application instance. */ @@ -369,14 +375,7 @@ public class ApplicationController { try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); Instance instance = application.get().require(job.application().instance()); - - Deployment deployment = instance.deployments().get(zone); - if ( zone.environment().isProduction() && deployment != null - && ( platform.compareTo(deployment.version()) < 0 && ! instance.change().isPinned() - || revision.compareTo(deployment.applicationVersion()) < 0 && ! (revision.isUnknown() && controller.system().isCd()))) - throw new IllegalArgumentException(String.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" + - " are older than the currently deployed (platform: %s, application: %s).", - job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion())); + rejectOldChange(instance, platform, revision, job, zone); if ( ! applicationPackage.trustedCertificates().isEmpty() && run.testerCertificate().isPresent()) @@ -403,21 +402,6 @@ public class ApplicationController { } } - private QuotaUsage deploymentQuotaUsage(ZoneId zoneId, ApplicationId applicationId) { - var application = configServer.nodeRepository().getApplication(zoneId, applicationId); - return DeploymentQuotaCalculator.calculateQuotaUsage(application); - } - - private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) { - return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone) - : applicationStore.get(application.tenant(), application.application(), revision)); - } - - /** Fetches the requested application package from the artifact store(s). */ - public ApplicationPackage getApplicationPackage(ApplicationId id, ApplicationVersion version) { - return new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)); - } - /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. */ public LockedApplication storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { applicationPackageValidator.validate(application.get(), applicationPackage, clock.instant()); @@ -683,6 +667,11 @@ public class ApplicationController { } } + /** Sets suspension status of the given deployment in its zone. */ + public void setSuspension(DeploymentId deploymentId, boolean suspend) { + configServer.setSuspension(deploymentId, suspend); + } + /** Deactivate application in the given zone */ public void deactivate(ApplicationId id, ZoneId zone) { lockApplicationOrThrow(TenantAndApplicationId.from(id), @@ -710,14 +699,6 @@ public class ApplicationController { public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; } - private TenantAndApplicationId dashToUnderscore(TenantAndApplicationId id) { - return TenantAndApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_")); - } - - private ApplicationId dashToUnderscore(ApplicationId id) { - return dashToUnderscore(TenantAndApplicationId.from(id)).instance(id.instance()); - } - /** * Returns a lock which provides exclusive rights to changing this application. * Any operation which stores an application need to first acquire this lock, then read, modify @@ -798,6 +779,38 @@ public class ApplicationController { } } + private void rejectOldChange(Instance instance, Version platform, ApplicationVersion revision, JobId job, ZoneId zone) { + Deployment deployment = instance.deployments().get(zone); + if (deployment == null) return; + if (!zone.environment().isProduction()) return; + + boolean platformIsOlder = platform.compareTo(deployment.version()) < 0 && !instance.change().isPinned(); + boolean revisionIsOlder = revision.compareTo(deployment.applicationVersion()) < 0 && + !(revision.isUnknown() && controller.system().isCd()); + if (platformIsOlder || revisionIsOlder) + throw new IllegalArgumentException(String.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" + + " are older than the currently deployed (platform: %s, application: %s).", + job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion())); + } + + private TenantAndApplicationId dashToUnderscore(TenantAndApplicationId id) { + return TenantAndApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_")); + } + + private ApplicationId dashToUnderscore(ApplicationId id) { + return dashToUnderscore(TenantAndApplicationId.from(id)).instance(id.instance()); + } + + private QuotaUsage deploymentQuotaUsage(ZoneId zoneId, ApplicationId applicationId) { + var application = configServer.nodeRepository().getApplication(zoneId, applicationId); + return DeploymentQuotaCalculator.calculateQuotaUsage(application); + } + + private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) { + return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone) + : applicationStore.get(application.tenant(), application.application(), revision)); + } + /* * Get the AthenzUser from this principal or Optional.empty if this does not represent a user. */ @@ -855,9 +868,4 @@ public class ApplicationController { return Map.copyOf(warnings); } - /** Sets suspension status of the given deployment in its zone. */ - public void setSuspension(DeploymentId deploymentId, boolean suspend) { - configServer.setSuspension(deploymentId, suspend); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index fc124947e5d..025b785a693 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; @@ -66,7 +67,10 @@ public class Instance { Instant instant, Map<DeploymentMetrics.Warning, Integer> warnings, QuotaUsage quotaUsage) { // Use info from previous deployment if available, otherwise create a new one. Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, applicationVersion, - version, instant)); + version, instant, + DeploymentMetrics.none, + DeploymentActivity.none, + QuotaUsage.none)); Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant, previousDeployment.metrics().with(warnings), previousDeployment.activity(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 800680d7327..3d17a7f8681 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -24,10 +24,6 @@ public class Deployment { private final DeploymentActivity activity; private final QuotaUsage quota; - public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) { - this(zone, applicationVersion, version, deployTime, DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none); - } - public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, DeploymentMetrics metrics, DeploymentActivity activity, QuotaUsage quota) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); @@ -76,6 +72,25 @@ public class Deployment { } @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Deployment that = (Deployment) o; + return zone.equals(that.zone) && + applicationVersion.equals(that.applicationVersion) && + version.equals(that.version) && + deployTime.equals(that.deployTime) && + metrics.equals(that.metrics) && + activity.equals(that.activity) && + quota.equals(that.quota); + } + + @Override + public int hashCode() { + return Objects.hash(zone, applicationVersion, version, deployTime, metrics, activity, quota); + } + + @Override public String toString() { return "deployment to " + zone + " of " + applicationVersion + " on version " + version + " at " + deployTime; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java index 03c08509a5e..71f0d64c43a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentActivity.java @@ -61,6 +61,19 @@ public class DeploymentActivity { activeRate(metrics.writesPerSecond(), lastWritesPerSecond)); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeploymentActivity that = (DeploymentActivity) o; + return lastQueried.equals(that.lastQueried) && lastWritten.equals(that.lastWritten) && lastQueriesPerSecond.equals(that.lastQueriesPerSecond) && lastWritesPerSecond.equals(that.lastWritesPerSecond); + } + + @Override + public int hashCode() { + return Objects.hash(lastQueried, lastWritten, lastQueriesPerSecond, lastWritesPerSecond); + } + public static DeploymentActivity create(Optional<Instant> queriedAt, Optional<Instant> writtenAt, OptionalDouble lastQueriesPerSecond, OptionalDouble lastWritesPerSecond) { if (queriedAt.isEmpty() && writtenAt.isEmpty()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java index 7a50184e7a4..094cb9a19b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java @@ -111,6 +111,25 @@ public class DeploymentMetrics { writeLatencyMills, instant, warnings); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeploymentMetrics that = (DeploymentMetrics) o; + return Double.compare(that.queriesPerSecond, queriesPerSecond) == 0 && + Double.compare(that.writesPerSecond, writesPerSecond) == 0 && + Double.compare(that.documentCount, documentCount) == 0 && + Double.compare(that.queryLatencyMillis, queryLatencyMillis) == 0 && + Double.compare(that.writeLatencyMills, writeLatencyMills) == 0 && + instant.equals(that.instant) && + warnings.equals(that.warnings); + } + + @Override + public int hashCode() { + return Objects.hash(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, writeLatencyMills, instant, warnings); + } + /** Types of deployment warnings. We currently have only one */ public enum Warning { all diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java index 13384b63c84..1e070d5a66b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/QuotaUsage.java @@ -1,12 +1,14 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import java.util.Objects; import java.util.OptionalDouble; /** * @author ogronnesby */ public class QuotaUsage { + public static final QuotaUsage none = new QuotaUsage(0.0); private final double rate; @@ -39,6 +41,19 @@ public class QuotaUsage { } @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QuotaUsage that = (QuotaUsage) o; + return Double.compare(that.rate, rate) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(rate); + } + + @Override public String toString() { return "QuotaUsage{" + "rate=" + rate + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 1a1b6988a96..b742c45bf09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceCon import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.Optional; @@ -22,6 +23,7 @@ import java.util.Optional; */ public enum SystemApplication { + controllerHost(ApplicationId.from("hosted-vespa", "controller-host", "default"), NodeType.controllerhost), configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), @@ -81,7 +83,12 @@ public enum SystemApplication { return Optional.of(Endpoint.of(this, zone, zoneRegistry.getConfigServerVipUri(zone))); } - /** All known system applications */ + /** All system applications that are not the controller */ + public static List<SystemApplication> notController() { + return List.copyOf(EnumSet.complementOf(EnumSet.of(SystemApplication.controllerHost))); + } + + /** All system applications */ public static List<SystemApplication> all() { return List.of(values()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java index 659edc96b77..11f36201f32 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java @@ -4,14 +4,19 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; public class ChangeManagementAssessor { @@ -28,10 +33,11 @@ public class ChangeManagementAssessor { Assessment assessmentInner(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes, ZoneId zone) { + List<String> impactedParentHosts = toParentHosts(impactedHostnames, allNodes); // Group impacted application nodes by parent host Map<NodeRepositoryNode, List<NodeRepositoryNode>> prParentHost = allNodes.stream() .filter(nodeRepositoryNode -> nodeRepositoryNode.getState() == NodeState.active) //TODO look at more states? - .filter(node -> impactedHostnames.contains(node.getParentHostname() == null ? "" : node.getParentHostname())) + .filter(node -> impactedParentHosts.contains(node.getParentHostname() == null ? "" : node.getParentHostname())) .collect(Collectors.groupingBy(node -> allNodes.stream() .filter(parent -> parent.getHostname().equals(node.getParentHostname())) @@ -39,32 +45,31 @@ public class ChangeManagementAssessor { )); // Group nodes pr cluster - Map<String, List<NodeRepositoryNode>> prCluster = prParentHost.values() + Map<Cluster, List<NodeRepositoryNode>> prCluster = prParentHost.values() .stream() .flatMap(Collection::stream) .collect(Collectors.groupingBy(ChangeManagementAssessor::clusterKey)); boolean allHostsReplacable = nodeRepository.isReplaceable( zone, - impactedHostnames.stream() - .map(HostName::from) + prParentHost.keySet().stream() + .filter(node -> node.getType() == NodeType.host) + .map(node -> HostName.from(node.getHostname())) .collect(Collectors.toList()) ); // Report assessment pr cluster var clusterAssessments = prCluster.entrySet().stream().map((entry) -> { - String key = entry.getKey(); + Cluster cluster = entry.getKey(); List<NodeRepositoryNode> nodes = entry.getValue(); - String app = Arrays.stream(key.split(":")).limit(3).collect(Collectors.joining(":")); - String cluster = Arrays.stream(key.split(":")).skip(3).collect(Collectors.joining(":")); - long[] totalStats = clusterStats(key, allNodes); - long[] impactedStats = clusterStats(key, nodes); + long[] totalStats = clusterStats(cluster, allNodes); + long[] impactedStats = clusterStats(cluster, nodes); ClusterAssessment assessment = new ClusterAssessment(); - assessment.app = app; + assessment.app = cluster.getApp(); assessment.zone = zone.value(); - assessment.cluster = cluster; + assessment.cluster = cluster.getClusterType() + ":" + cluster.getClusterId(); assessment.clusterSize = totalStats[0]; assessment.clusterImpact = impactedStats[0]; assessment.groupsTotal = totalStats[1]; @@ -76,7 +81,7 @@ public class ChangeManagementAssessor { // TODO do some heuristic on suggestion action assessment.suggestedAction = allHostsReplacable ? "Retire all hosts" : "nothing"; // TODO do some heuristic on impact - assessment.impact = "na"; + assessment.impact = getImpact(cluster, impactedStats, totalStats); return assessment; }).collect(Collectors.toList()); @@ -99,21 +104,83 @@ public class ChangeManagementAssessor { return new Assessment(clusterAssessments, hostAssessments); } - private static String clusterKey(NodeRepositoryNode node) { - if (node.getOwner() != null && node.getMembership() != null) { - String appId = String.format("%s:%s:%s", node.getOwner().tenant, node.getOwner().application, node.getOwner().instance); - String cluster = String.format("%s:%s", node.getMembership().clustertype, node.getMembership().clusterid); - return appId + ":" + cluster; - } - return ""; + private List<String> toParentHosts(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes) { + return impactedHostnames.stream() + .map(hostname -> + allNodes.stream() + .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.getType())) + .filter(node -> hostname.equals(node.getHostname()) || hostname.equals(node.getParentHostname())) + .map(node -> { + if (node.getType() == NodeType.host) + return node.getHostname(); + return node.getParentHostname(); + }).findFirst().orElseThrow() + ) + .collect(Collectors.toList()); } - private static long[] clusterStats(String key, List<NodeRepositoryNode> containerNodes) { - List<NodeRepositoryNode> clusterNodes = containerNodes.stream().filter(nodeRepositoryNode -> clusterKey(nodeRepositoryNode).equals(key)).collect(Collectors.toList()); + private static Cluster clusterKey(NodeRepositoryNode node) { + String appId = String.format("%s:%s:%s", node.getOwner().tenant, node.getOwner().application, node.getOwner().instance); + return new Cluster(Node.ClusterType.valueOf(node.getMembership().clustertype), node.getMembership().clusterid, appId, node.getType()); + } + + private static long[] clusterStats(Cluster cluster, List<NodeRepositoryNode> containerNodes) { + List<NodeRepositoryNode> clusterNodes = containerNodes.stream().filter(nodeRepositoryNode -> cluster.equals(clusterKey(nodeRepositoryNode))).collect(Collectors.toList()); long groups = clusterNodes.stream().map(nodeRepositoryNode -> nodeRepositoryNode.getMembership() != null ? nodeRepositoryNode.getMembership().group : "").distinct().count(); return new long[] { clusterNodes.size(), groups}; } + private String getImpact(Cluster cluster, long[] impactedStats, long[] totalStats) { + switch (cluster.getNodeType()) { + case tenant: + return getTenantImpact(cluster, impactedStats, totalStats); + case proxy: + return getProxyImpact(impactedStats[0], totalStats[0]); + case config: + return getConfigServerImpact(impactedStats[0]); + default: + return "Unkown impact"; + } + } + + private String getTenantImpact(Cluster cluster, long[] impactedStats, long[] totalStats) { + switch (cluster.getClusterType()) { + case container: + return getContainerImpact(impactedStats[0], totalStats[0]); + case content: + case combined: + return getContentImpact(totalStats[1] > 1, impactedStats[0], impactedStats[1]); + default: + return "Unknown impact"; + } + } + + private String getProxyImpact(long impactedNodes, long totalNodes) { + int impact = (int) (100.0 * impactedNodes / totalNodes); + return impact + "% of routing nodes impacted. Consider reprovisioning if too many"; + } + + private String getConfigServerImpact(long impactedNodes) { + if (impactedNodes == 1) { + return "Acceptable impact"; + } + return "Large impact. Consider reprovisioning one or more config servers"; + } + + private String getContainerImpact(long impactedNodes, long totalNodes) { + if ((double) impactedNodes / totalNodes <= 0.1) { + return "Impact not larger than upgrade policy"; + } + return "Impact larger than upgrade policy"; + } + + private String getContentImpact(boolean isGrouped, long impactedNodes, long impactedGroups) { + if ((isGrouped && impactedGroups == 1) || impactedNodes == 1) + return "Impact not larger than upgrade policy"; + return "Impact larger than upgrade policy"; + } + + public static class Assessment { List<ClusterAssessment> clusterAssessments; List<HostAssessment> hostAssessments; @@ -152,4 +219,49 @@ public class ChangeManagementAssessor { public int numberOfProblematicChildren; } + private static class Cluster { + private Node.ClusterType clusterType; + private String clusterId; + private String app; + private NodeType nodeType; + + public Cluster(Node.ClusterType clusterType, String clusterId, String app, NodeType nodeType) { + this.clusterType = clusterType; + this.clusterId = clusterId; + this.app = app; + this.nodeType = nodeType; + } + + public Node.ClusterType getClusterType() { + return clusterType; + } + + public String getClusterId() { + return clusterId; + } + + public String getApp() { + return app; + } + + public NodeType getNodeType() { + return nodeType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Cluster cluster = (Cluster) o; + return Objects.equals(clusterType, cluster.clusterType) && + Objects.equals(clusterId, cluster.clusterId) && + Objects.equals(app, cluster.app); + } + + @Override + public int hashCode() { + return Objects.hash(clusterType, clusterId, app); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java index 80a79d004c6..0f976458257 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java @@ -41,7 +41,7 @@ public class ContainerImageExpirer extends ControllerMaintainer { .filter(image -> canExpire(image, now, versionStatus)) .collect(Collectors.toList()); if (!imagesToExpire.isEmpty()) { - log.log(Level.INFO, "Expiring container images: " + imagesToExpire); + log.log(Level.INFO, "Expiring " + imagesToExpire.size() + " container images: " + imagesToExpire); controller().serviceRegistry().containerRegistry().deleteAll(imagesToExpire); } return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 82dfd9d65bf..8433afaf006 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -143,7 +143,7 @@ public class ControllerMaintenance extends AbstractComponent { this.resourceTagMaintainer = duration(30, MINUTES); this.systemRoutingPolicyMaintainer = duration(10, MINUTES); this.applicationMetaDataGarbageCollector = duration(12, HOURS); - this.containerImageExpirer = duration(2, HOURS); + this.containerImageExpirer = duration(12, HOURS); this.hostSwitchUpdater = duration(12, HOURS); this.reindexingTriggerer = duration(1, HOURS); this.endpointCertificateMaintainer = duration(12, HOURS); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 37de7369452..e5316788802 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -1,13 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.yolean.Exceptions; import java.time.Duration; +import java.util.Optional; import java.util.logging.Level; /** @@ -28,7 +33,7 @@ public class DeploymentExpirer extends ControllerMaintainer { for (Application application : controller().applications().readable()) { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) { - if (!isExpired(deployment)) continue; + if (!isExpired(deployment, instance.id())) continue; try { log.log(Level.INFO, "Expiring deployment of " + instance.id() + " in " + deployment.zone()); @@ -45,10 +50,19 @@ public class DeploymentExpirer extends ControllerMaintainer { } /** Returns whether given deployment has expired according to its TTL */ - private boolean isExpired(Deployment deployment) { + private boolean isExpired(Deployment deployment, ApplicationId instance) { if (deployment.zone().environment().isProduction()) return false; // Never expire production deployments - return controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()) - .map(timeToLive -> deployment.at().plus(timeToLive).isBefore(controller().clock().instant())) + + Optional<Duration> ttl = controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()); + if (ttl.isEmpty()) return false; + + Optional<JobId> jobId = JobType.from(controller().system(), deployment.zone()) + .map(type -> new JobId(instance, type)); + if (jobId.isEmpty()) return false; + + return controller().jobController().last(jobId.get()) + .flatMap(Run::end) + .map(end -> end.plus(ttl.get()).isBefore(controller().clock().instant())) .orElse(false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 7952355d5fb..9859d12510a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -14,6 +14,7 @@ import java.time.Duration; import java.util.Comparator; import java.util.EnumSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.logging.Logger; @@ -28,15 +29,18 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain private static final Logger log = Logger.getLogger(InfrastructureUpgrader.class.getName()); protected final UpgradePolicy upgradePolicy; + private final List<SystemApplication> managedApplications; - public InfrastructureUpgrader(Controller controller, Duration interval, UpgradePolicy upgradePolicy, String name) { + public InfrastructureUpgrader(Controller controller, Duration interval, UpgradePolicy upgradePolicy, + List<SystemApplication> managedApplications, String name) { super(controller, interval, name, EnumSet.allOf(SystemName.class)); this.upgradePolicy = upgradePolicy; + this.managedApplications = List.copyOf(Objects.requireNonNull(managedApplications)); } @Override protected boolean maintain() { - targetVersion().ifPresent(target -> upgradeAll(target, SystemApplication.all())); + targetVersion().ifPresent(target -> upgradeAll(target, managedApplications)); return true; } @@ -101,7 +105,7 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain try { return controller().serviceRegistry().configServer() .nodeRepository() - .list(zone.getId(), application.id()) + .list(zone.getVirtualId(), application.id()) .stream() .filter(node -> expectUpgradeOf(node, application, zone)) .map(versionField) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 43e9ce51040..79bea294472 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -37,7 +37,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { private final CloudName cloud; public OsUpgrader(Controller controller, Duration interval, CloudName cloud) { - super(controller, interval, controller.zoneRegistry().osUpgradePolicy(cloud), name(cloud)); + super(controller, interval, controller.zoneRegistry().osUpgradePolicy(cloud), SystemApplication.all(), name(cloud)); this.cloud = cloud; } @@ -47,9 +47,9 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { .map(totalBudget -> zoneBudgetOf(totalBudget, zone)); log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s%s", application.id(), target.osVersion().version().toFullString(), - zone.getId(), zone.getCloudName(), + zone.getVirtualId(), zone.getCloudName(), zoneUpgradeBudget.map(d -> " with time budget " + d).orElse(""))); - controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getId(), application.nodeType(), + controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getVirtualId(), application.nodeType(), target.osVersion().version(), zoneUpgradeBudget); } @@ -78,7 +78,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { protected boolean changeTargetTo(OsVersionTarget target, SystemApplication application, ZoneApi zone) { if (!application.shouldUpgradeOs()) return false; return controller().serviceRegistry().configServer().nodeRepository() - .targetVersionsOf(zone.getId()) + .targetVersionsOf(zone.getVirtualId()) .osVersion(application.nodeType()) .map(currentTarget -> target.osVersion().version().isAfter(currentTarget)) .orElse(true); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index b84e77a1d85..e286db5882b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -26,7 +26,7 @@ public class SystemUpgrader extends InfrastructureUpgrader<Version> { private static final Set<Node.State> upgradableNodeStates = Set.of(Node.State.active, Node.State.reserved); public SystemUpgrader(Controller controller, Duration interval) { - super(controller, interval, controller.zoneRegistry().upgradePolicy(), null); + super(controller, interval, controller.zoneRegistry().upgradePolicy(), SystemApplication.notController(), null); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 52fbe26bd7c..24b553e5153 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -1,10 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.hash.Hashing; -import com.google.common.util.concurrent.UncheckedExecutionException; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; @@ -53,7 +49,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.concurrent.ExecutionException; /** * Serializes {@link Application}s to/from slime. @@ -297,7 +292,7 @@ public class ApplicationSerializer { Inspector root = slime.get(); TenantAndApplicationId id = TenantAndApplicationId.fromSerialized(root.field(idField).asString()); - Instant createdAt = Instant.ofEpochMilli(root.field(createdAtField).asLong()); + Instant createdAt = Serializers.instant(root.field(createdAtField)); DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString(), false); ValidationOverrides validationOverrides = ValidationOverrides.fromXml(root.field(validationOverridesField).asString()); Optional<IssueId> deploymentIssueId = Serializers.optionalString(root.field(deploymentIssueField)).map(IssueId::from); @@ -356,7 +351,7 @@ public class ApplicationSerializer { return new Deployment(zoneIdFromSlime(deploymentObject.field(zoneField)), applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)), Version.fromString(deploymentObject.field(versionField).asString()), - Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), + Serializers.instant(deploymentObject.field(deployTimeField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)), Serializers.optionalInstant(deploymentObject.field(lastWrittenField)), @@ -366,9 +361,7 @@ public class ApplicationSerializer { } private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { - Optional<Instant> instant = object.field(deploymentMetricsUpdateTime).valid() ? - Optional.of(Instant.ofEpochMilli(object.field(deploymentMetricsUpdateTime).asLong())) : - Optional.empty(); + Optional<Instant> instant = Serializers.optionalInstant(object.field(deploymentMetricsUpdateTime)); return new DeploymentMetrics(object.field(deploymentMetricsQPSField).asDouble(), object.field(deploymentMetricsWPSField).asDouble(), object.field(deploymentMetricsDocsField).asDouble(), @@ -391,7 +384,7 @@ public class ApplicationSerializer { object.traverse((ArrayTraverser) (idx, statusObject) -> statusMap.put(new RotationId(statusObject.field(rotationIdField).asString()), new RotationStatus.Targets( singleRotationStatusFromSlime(statusObject.field(statusField)), - Instant.ofEpochMilli(statusObject.field(lastUpdatedField).asLong())))); + Serializers.instant(statusObject.field(lastUpdatedField))))); return RotationStatus.from(statusMap); } @@ -440,7 +433,7 @@ public class ApplicationSerializer { object.field(jobStatusField).traverse((ArrayTraverser) (__, jobPauseObject) -> JobType.fromOptionalJobName(jobPauseObject.field(jobTypeField).asString()) .ifPresent(jobType -> jobPauses.put(jobType, - Instant.ofEpochMilli(jobPauseObject.field(pausedUntilField).asLong())))); + Serializers.instant(jobPauseObject.field(pausedUntilField))))); return jobPauses; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index b411f460568..7ea722bf5de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -7,7 +7,6 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; -import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -52,7 +51,7 @@ public class AuditLogSerializer { Cursor root = slime.get(); root.field(entriesField).traverse((ArrayTraverser) (i, entryObject) -> { entries.add(new AuditLog.Entry( - Instant.ofEpochMilli(entryObject.field(atField).asLong()), + Serializers.instant(entryObject.field(atField)), entryObject.field(principalField).asString(), methodFrom(entryObject.field(methodField)), entryObject.field(resourceField).asString(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java index 0e8b6087901..30fcc0e40c6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java @@ -5,8 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; -import java.time.Instant; - /** * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.ControllerVersion}. * @@ -38,7 +36,7 @@ public class ControllerVersionSerializer { var root = slime.get(); var version = Version.fromString(root.field(VERSION_FIELD).asString()); var commitSha = root.field(COMMIT_SHA_FIELD).asString(); - var commitDate = Instant.ofEpochMilli(root.field(COMMIT_DATE_FIELD).asLong()); + var commitDate = Serializers.instant(root.field(COMMIT_DATE_FIELD)); return new ControllerVersion(version, commitSha, commitDate); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index fffe781e6e1..6416d077ce4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; import java.io.IOException; import java.io.UncheckedIOException; -import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -93,7 +92,7 @@ class LogSerializer { private LogEntry fromSlime(Inspector entryObject) { return new LogEntry(entryObject.field(idField).asLong(), - Instant.ofEpochMilli(entryObject.field(timestampField).asLong()), + Serializers.instant(entryObject.field(timestampField)), typeOf(entryObject.field(typeField).asString()), entryObject.field(messageField).asString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 1aa229984a8..0ecd86a4a38 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -121,9 +121,7 @@ class RunSerializer { // For historical reasons are the step details stored in a separate JSON structure from the step statuses. Inspector stepDetailsField = detailsField.field(step); Inspector startTimeValue = stepDetailsField.field(startTimeField); - Optional<Instant> startTime = startTimeValue.valid() ? - Optional.of(instantOf(startTimeValue.asLong())) : - Optional.empty(); + Optional<Instant> startTime = Serializers.optionalInstant(startTimeValue); steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); }); @@ -132,7 +130,7 @@ class RunSerializer { runObject.field(numberField).asLong()), steps, versionsFromSlime(runObject.field(versionsField)), - Instant.ofEpochMilli(runObject.field(startField).asLong()), + Serializers.instant(runObject.field(startField)), Serializers.optionalInstant(runObject.field(endField)), runStatusOf(runObject.field(statusField).asString()), runObject.field(lastTestRecordField).asLong(), @@ -259,7 +257,7 @@ class RunSerializer { applicationVersion.commit().ifPresent(commit -> versionsObject.setString(commitField, commit)); } - // Don't change this — introduce a separate array with new values if needed. + // Don't change this - introduce a separate array with new values if needed. private void toSlime(ConvergenceSummary summary, Cursor summaryArray) { summaryArray.addLong(summary.nodes()); summaryArray.addLong(summary.down()); @@ -341,10 +339,6 @@ class RunSerializer { return instant.toEpochMilli(); } - static Instant instantOf(Long epochMillis) { - return Instant.ofEpochMilli(epochMillis); - } - static String valueOf(RunStatus status) { switch (status) { case running : return "running"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java index b254732f324..7c8a09e244e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java @@ -20,6 +20,10 @@ public class Serializers { private Serializers() {} + public static Instant instant(Inspector field) { + return Instant.ofEpochMilli(field.asLong()); + } + public static OptionalLong optionalLong(Inspector field) { return field.valid() ? OptionalLong.of(field.asLong()) : OptionalLong.empty(); } @@ -37,13 +41,11 @@ public class Serializers { } public static Optional<Instant> optionalInstant(Inspector field) { - var value = optionalLong(field); - return value.isPresent() ? Optional.of(Instant.ofEpochMilli(value.getAsLong())) : Optional.empty(); + return optionalLong(field).stream().mapToObj(Instant::ofEpochMilli).findFirst(); } public static Optional<Duration> optionalDuration(Inspector field) { - var value = optionalLong(field); - return value.isPresent() ? Optional.of(Duration.ofMillis(value.getAsLong())) : Optional.empty(); + return optionalLong(field).stream().mapToObj(Duration::ofMillis).findFirst(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 8cb87b8a72d..8e97368624d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -151,14 +151,14 @@ public class TenantSerializer { Property property = new Property(tenantObject.field(propertyField).asString()); Optional<PropertyId> propertyId = SlimeUtils.optionalString(tenantObject.field(propertyIdField)).map(PropertyId::new); Optional<Contact> contact = contactFrom(tenantObject.field(contactField)); - Instant createdAt = Instant.ofEpochMilli(tenantObject.field(createdAtField).asLong()); + Instant createdAt = Serializers.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo); } private CloudTenant cloudTenantFrom(Inspector tenantObject) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); - Instant createdAt = Instant.ofEpochMilli(tenantObject.field(createdAtField).asLong()); + Instant createdAt = Serializers.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); Optional<Principal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); @@ -227,7 +227,7 @@ public class TenantSerializer { private LastLoginInfo lastLoginInfoFromSlime(Inspector lastLoginInfoObject) { Map<LastLoginInfo.UserLevel, Instant> lastLoginByUserLevel = new HashMap<>(); lastLoginInfoObject.traverse((String name, Inspector value) -> - lastLoginByUserLevel.put(userLevelOf(name), Instant.ofEpochMilli(value.asLong()))); + lastLoginByUserLevel.put(userLevelOf(name), Serializers.instant(value))); return new LastLoginInfo(lastLoginByUserLevel); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 6eb5b8fadcd..12d15aa7cdd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -7,19 +7,14 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * Serializer for {@link VersionStatus}. @@ -111,7 +106,7 @@ public class VersionStatusSerializer { var version = Version.fromString(object.field(deploymentStatisticsField).field(versionField).asString()); return new VespaVersion(version, object.field(releaseCommitField).asString(), - Instant.ofEpochMilli(object.field(committedAtField).asLong()), + Serializers.instant(object.field(committedAtField)), object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), object.field(isReleasedField).asBool(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index f5dcae9c961..858bf857429 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import ai.vespa.util.http.retry.Sleeper; +import ai.vespa.util.http.hc4.retry.Sleeper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index ca080078328..81183ac9aca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -50,6 +50,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; @@ -816,9 +817,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { nodeObject.setString("version", node.currentVersion().toString()); nodeObject.setString("flavor", node.flavor()); toSlime(node.resources(), nodeObject); - nodeObject.setBool("fastDisk", node.resources().diskSpeed() == NodeResources.DiskSpeed.fast); // TODO: Remove nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); + nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.getEvent()))); + nodeObject.setBool("retired", node.retired() || node.wantToRetire()); + nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); + nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); } return new SlimeJsonResponse(slime); } @@ -1557,7 +1561,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ZoneId zone = requireZone(environment, region); ServiceApiResponse response = new ServiceApiResponse(zone, new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.zoneRegistry().getConfigServerApiUris(zone), + List.of(controller.zoneRegistry().getConfigServerVipUri(zone)), request.getUri()); response.setResponse(applicationView); return response; @@ -1575,7 +1579,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), deploymentId.applicationId(), - controller.zoneRegistry().getConfigServerApiUris(deploymentId.zoneId()), + List.of(controller.zoneRegistry().getConfigServerVipUri(deploymentId.zoneId())), request.getUri()); response.setResponse(result, serviceName, restPath); return response; @@ -1974,8 +1978,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { keyObject.setString("user", user.getName()); }); + // TODO: remove this once console is updated toSlime(object, cloudTenant.tenantSecretStores()); + toSlime(object.setObject("integrations").setObject("aws"), + controller.serviceRegistry().roleService().getTenantRole(tenant.name()), + cloudTenant.tenantSecretStores()); + var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name()); var usedQuota = applications.stream() .map(Application::quotaUsage) @@ -2249,13 +2258,24 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void toSlime(Cursor object, List<TenantSecretStore> tenantSecretStores) { Cursor secretStore = object.setArray("secretStores"); tenantSecretStores.forEach(store -> { - Cursor storeObject = secretStore.addObject(); - storeObject.setString("name", store.getName()); - storeObject.setString("awsId", store.getAwsId()); - storeObject.setString("role", store.getRole()); + toSlime(secretStore.addObject(), store); + }); + } + + private void toSlime(Cursor object, TenantRoles tenantRoles, List<TenantSecretStore> tenantSecretStores) { + object.setString("tenantRole", tenantRoles.containerRole()); + var stores = object.setArray("accounts"); + tenantSecretStores.forEach(secretStore -> { + toSlime(stores.addObject(), secretStore); }); } + private void toSlime(Cursor object, TenantSecretStore secretStore) { + object.setString("name", secretStore.getName()); + object.setString("awsId", secretStore.getAwsId()); + object.setString("role", secretStore.getRole()); + } + private String readToString(InputStream stream) { Scanner scanner = new Scanner(stream).useDelimiter("\\A"); if ( ! scanner.hasNext()) return null; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 1bb3b1c5de8..8b5280a0e8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -31,17 +31,18 @@ import java.util.stream.Stream; @SuppressWarnings("unused") public class ConfigServerApiHandler extends AuditLoggingRequestHandler { - private static final ZoneId CONTROLLER_ZONE = ZoneId.from("prod", "controller"); private static final URI CONTROLLER_URI = URI.create("https://localhost:4443/"); private static final List<String> WHITELISTED_APIS = List.of("/flags/v1/", "/nodes/v2/", "/orchestrator/v1/"); private final ZoneRegistry zoneRegistry; private final ConfigServerRestExecutor proxy; + private final ZoneId controllerZone; public ConfigServerApiHandler(Context parentCtx, ServiceRegistry serviceRegistry, ConfigServerRestExecutor proxy, Controller controller) { super(parentCtx, controller.auditLogger()); this.zoneRegistry = serviceRegistry.zoneRegistry(); + this.controllerZone = zoneRegistry.systemZone().getVirtualId(); this.proxy = proxy; } @@ -83,7 +84,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { } ZoneId zoneId = ZoneId.from(path.get("environment"), path.get("region")); - if (! zoneRegistry.hasZone(zoneId) && ! CONTROLLER_ZONE.equals(zoneId)) { + if (! zoneRegistry.hasZone(zoneId) && ! controllerZone.equals(zoneId)) { throw new IllegalArgumentException("No such zone: " + zoneId.value()); } @@ -102,7 +103,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { ZoneList zoneList = zoneRegistry.zones().reachable(); Cursor zones = root.setArray("zones"); - Stream.concat(Stream.of(CONTROLLER_ZONE), zoneRegistry.zones().reachable().ids().stream()) + Stream.concat(Stream.of(controllerZone), zoneRegistry.zones().reachable().ids().stream()) .forEach(zone -> { Cursor object = zones.addObject(); object.setString("environment", zone.environment().value()); @@ -118,7 +119,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { } private URI getEndpoint(ZoneId zoneId) { - return CONTROLLER_ZONE.equals(zoneId) ? CONTROLLER_URI : zoneRegistry.getConfigServerVipUri(zoneId); + return controllerZone.equals(zoneId) ? CONTROLLER_URI : zoneRegistry.getConfigServerVipUri(zoneId); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java index 161d3734aae..1709e3cb65f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.systemflags; -import ai.vespa.util.http.retry.DelayedConnectionLevelRetryHandler; +import ai.vespa.util.http.hc4.retry.DelayedConnectionLevelRetryHandler; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.athenz.api.AthenzIdentity; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 226852f1f3d..8fd5f07b9ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -65,14 +65,14 @@ public class OsVersionStatus { for (var zone : zonesToUpgrade(controller)) { if (!application.shouldUpgradeOs()) continue; var targetOsVersion = controller.serviceRegistry().configServer().nodeRepository() - .targetVersionsOf(zone.getId()) + .targetVersionsOf(zone.getVirtualId()) .osVersion(application.nodeType()) .orElse(Version.emptyVersion); - for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id())) { + for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getVirtualId(), application.id())) { if (!OsUpgrader.canUpgrade(node)) continue; var suspendedAt = node.suspendedSince(); - var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), + var nodeVersion = new NodeVersion(node.hostname(), zone.getVirtualId(), node.currentOsVersion(), targetOsVersion, suspendedAt); var osVersion = new OsVersion(nodeVersion.currentVersion(), zone.getCloudName()); osVersions.putIfAbsent(osVersion, new ArrayList<>()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index a30409dfa80..625154693da 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -152,7 +152,7 @@ public class VersionStatus { private static NodeVersions findSystemApplicationVersions(Controller controller, VersionStatus versionStatus) { var nodeVersions = new LinkedHashMap<HostName, NodeVersion>(); for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { - for (var application : SystemApplication.all()) { + for (var application : SystemApplication.notController()) { var nodes = controller.serviceRegistry().configServer().nodeRepository() .list(zone.getId(), application.id()).stream() .filter(SystemUpgrader::eligibleForUpgrade) diff --git a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def index ddaa1e635db..1a163ba1fff 100644 --- a/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def +++ b/controller-server/src/main/resources/configdefinitions/vespa.hosted.controller.tls.config.tls.def @@ -5,5 +5,5 @@ namespace=vespa.hosted.controller.tls.config caTrustStore string # Secret store key names for certificate and private key -certificateSecret string default=vespa_hosted.tls.cert -privateKeySecret string default=vespa_hosted.tls.key +certificateSecret string +privateKeySecret string diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index f8645139244..03487163936 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -214,7 +214,7 @@ public final class ControllerTester { /** Upgrade system applications in all zones to given version */ public void upgradeSystemApplications(Version version) { - upgradeSystemApplications(version, SystemApplication.all()); + upgradeSystemApplications(version, SystemApplication.notController()); } /** Upgrade given system applications in all zones to version */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index a7a99f286df..6c43c4e120d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; @@ -101,7 +100,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { - bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.all()); + bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.notController()); } /** Sets the ConfigChangeActions that will be returned on next deployment. */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java index af0e3d5807d..3d87f2d7190 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -14,18 +14,24 @@ import java.util.Objects; * @author hakonhall */ public class ZoneApiMock implements ZoneApi { + private final SystemName systemName; private final ZoneId id; + private final ZoneId virtualId; private final CloudName cloudName; private final String cloudNativeRegionName; public static Builder newBuilder() { return new Builder(); } - private ZoneApiMock(SystemName systemName, ZoneId id, CloudName cloudName, String cloudNativeRegionName) { + private ZoneApiMock(SystemName systemName, ZoneId id, ZoneId virtualId, CloudName cloudName, String cloudNativeRegionName) { this.systemName = systemName; this.id = id; + this.virtualId = virtualId; this.cloudName = cloudName; this.cloudNativeRegionName = cloudNativeRegionName; + if (virtualId != null && virtualId.equals(id)) { + throw new IllegalArgumentException("Virtual ID cannot be equal to zone ID: " + id); + } } public static ZoneApiMock fromId(String id) { @@ -47,6 +53,11 @@ public class ZoneApiMock implements ZoneApi { public ZoneId getId() { return id; } @Override + public ZoneId getVirtualId() { + return virtualId == null ? getId() : virtualId; + } + + @Override public CloudName getCloudName() { return cloudName; } @Override @@ -66,8 +77,11 @@ public class ZoneApiMock implements ZoneApi { } public static class Builder { + private final SystemName systemName = SystemName.defaultSystem(); + private ZoneId id = ZoneId.defaultId(); + private ZoneId virtualId ; private CloudName cloudName = CloudName.defaultName(); private String cloudNativeRegionName = id.region().value(); @@ -78,6 +92,15 @@ public class ZoneApiMock implements ZoneApi { public Builder withId(String id) { return with(ZoneId.from(id)); } + public Builder withVirtualId(ZoneId virtualId) { + this.virtualId = virtualId; + return this; + } + + public Builder withVirtualId(String virtualId) { + return withVirtualId(ZoneId.from(virtualId)); + } + public Builder with(CloudName cloudName) { this.cloudName = cloudName; return this; @@ -91,7 +114,8 @@ public class ZoneApiMock implements ZoneApi { } public ZoneApiMock build() { - return new ZoneApiMock(systemName, id, cloudName, cloudNativeRegionName); + return new ZoneApiMock(systemName, id, virtualId, cloudName, cloudNativeRegionName); } } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 8fcd9527517..1d2c743ffba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -141,6 +141,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public ZoneApi systemZone() { + return ZoneApiMock.fromId("prod.controller"); + } + + @Override public ZoneFilter zones() { return ZoneFilterMock.from(zones, zoneRoutingMethods, reprovisionToUpgradeOs); } @@ -216,25 +221,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override - public List<URI> getConfigServerUris(ZoneId zoneId) { - return List.of( - URI.create(String.format("https://cfg1.%s.test:4443/", zoneId.value())), - URI.create(String.format("https://cfg2.%s.test:4443/", zoneId.value()))); - } - - @Override public URI getConfigServerVipUri(ZoneId zoneId) { return URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value())); } @Override - public List<URI> getConfigServerApiUris(ZoneId zoneId) { - return List.of( - URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value())), - URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value()))); - } - - @Override public Optional<Duration> getDeploymentTimeToLive(ZoneId zoneId) { return Optional.ofNullable(deploymentTimeToLive.get(zoneId)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java index 8a91be2b2a3..575a38cd637 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMemb import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; @@ -39,16 +40,16 @@ public class ChangeManagementAssessorTest { ZoneId zone = ZoneId.from("prod", "eu-trd"); List<String> hostNames = Collections.singletonList("host1"); List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); - allNodesInZone.add(createNode("node1", "host1", "myapp", "default", 0 )); - allNodesInZone.add(createNode("node2", "host1", "myapp", "default", 0 )); - allNodesInZone.add(createNode("node3", "host1", "myapp", "default", 0 )); + allNodesInZone.add(createNode("node1", "host1", "default", 0 )); + allNodesInZone.add(createNode("node2", "host1", "default", 0 )); + allNodesInZone.add(createNode("node3", "host1", "default", 0 )); // Add an not impacted hosts - allNodesInZone.add(createNode("node4", "host2", "myapp", "default", 0 )); + allNodesInZone.add(createNode("node4", "host2", "default", 0 )); // Add tenant hosts - allNodesInZone.add(createHost("host1", "switch1")); - allNodesInZone.add(createHost("host2", "switch1")); + allNodesInZone.add(createHost("host1", NodeType.host)); + allNodesInZone.add(createHost("host2", NodeType.host)); // Make Assessment List<ChangeManagementAssessor.ClusterAssessment> assessments @@ -72,25 +73,26 @@ public class ChangeManagementAssessorTest { List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); // Two impacted nodes on host1 - allNodesInZone.add(createNode("node1", "host1", "myapp", "default", 0 )); - allNodesInZone.add(createNode("node2", "host1", "myapp", "default", 0 )); + allNodesInZone.add(createNode("node1", "host1", "default", 0 )); + allNodesInZone.add(createNode("node2", "host1", "default", 0 )); // One impacted nodes on host2 - allNodesInZone.add(createNode("node3", "host2", "myapp", "default", 0 )); + allNodesInZone.add(createNode("node3", "host2", "default", 0 )); // Another group on hosts not impacted - allNodesInZone.add(createNode("node4", "host3", "myapp", "default", 1 )); - allNodesInZone.add(createNode("node5", "host3", "myapp", "default", 1 )); - allNodesInZone.add(createNode("node6", "host3", "myapp", "default", 1 )); + allNodesInZone.add(createNode("node4", "host3", "default", 1 )); + allNodesInZone.add(createNode("node5", "host3", "default", 1 )); + allNodesInZone.add(createNode("node6", "host3", "default", 1 )); // Another cluster on hosts not impacted - this one also with three different groups (should all be ignored here) - allNodesInZone.add(createNode("node4", "host4", "myapp", "myman", 4 )); - allNodesInZone.add(createNode("node5", "host4", "myapp", "myman", 5 )); - allNodesInZone.add(createNode("node6", "host4", "myapp", "myman", 6 )); + allNodesInZone.add(createNode("node4", "host4", "myman", 4 )); + allNodesInZone.add(createNode("node5", "host4", "myman", 5 )); + allNodesInZone.add(createNode("node6", "host4", "myman", 6 )); // Add tenant hosts - allNodesInZone.add(createHost("host1", "switch1")); - allNodesInZone.add(createHost("host2", "switch1")); + allNodesInZone.add(createHost("host1", NodeType.host)); + allNodesInZone.add(createHost("host2", NodeType.host)); + // Make Assessment ChangeManagementAssessor.Assessment assessment @@ -106,6 +108,7 @@ public class ChangeManagementAssessorTest { assertEquals("content:default", clusterAssessments.get(0).cluster); assertEquals("mytenant:myapp:default", clusterAssessments.get(0).app); assertEquals("prod.eu-trd", clusterAssessments.get(0).zone); + assertEquals("Impact not larger than upgrade policy", clusterAssessments.get(0).impact); List<ChangeManagementAssessor.HostAssessment> hostAssessments = assessment.getHostAssessments(); assertEquals(2, hostAssessments.size()); @@ -117,11 +120,47 @@ public class ChangeManagementAssessorTest { )); } - private NodeOwner createOwner(String tenant, String application, String instance) { + @Test + public void two_config_nodes() { + var zone = ZoneId.from("prod", "eu-trd"); + var hostNames = Arrays.asList("config1", "config2"); + var allNodesInZone = new ArrayList<NodeRepositoryNode>(); + + // Add config nodes and parents + allNodesInZone.add(createNode("config1", "confighost1", "config", 0, NodeType.config)); + allNodesInZone.add(createHost("confighost1", NodeType.confighost)); + allNodesInZone.add(createNode("config2", "confighost2", "config", 0, NodeType.config)); + allNodesInZone.add(createHost("confighost2", NodeType.confighost)); + + var assessment = changeManagementAssessor.assessmentInner(hostNames, allNodesInZone, zone).getClusterAssessments(); + var configAssessment = assessment.get(0); + assertEquals("Large impact. Consider reprovisioning one or more config servers", configAssessment.impact); + assertEquals(2, configAssessment.clusterImpact); + } + + @Test + public void one_of_three_proxy_nodes() { + var zone = ZoneId.from("prod", "eu-trd"); + var hostNames = Arrays.asList("routing1"); + var allNodesInZone = new ArrayList<NodeRepositoryNode>(); + + // Add routing nodes and parents + allNodesInZone.add(createNode("routing1", "parentrouting1", "routing", 0, NodeType.proxy)); + allNodesInZone.add(createHost("parentrouting1", NodeType.proxyhost)); + allNodesInZone.add(createNode("routing2", "parentrouting2", "routing", 0, NodeType.proxy)); + allNodesInZone.add(createHost("parentrouting2", NodeType.proxyhost)); + allNodesInZone.add(createNode("routing3", "parentrouting3", "routing", 0, NodeType.proxy)); + allNodesInZone.add(createHost("parentrouting3", NodeType.proxyhost)); + + var assessment = changeManagementAssessor.assessmentInner(hostNames, allNodesInZone, zone).getClusterAssessments(); + assertEquals("33% of routing nodes impacted. Consider reprovisioning if too many", assessment.get(0).impact); + } + + private NodeOwner createOwner() { NodeOwner owner = new NodeOwner(); - owner.tenant = tenant; - owner.application = application; - owner.instance = instance; + owner.tenant = "mytenant"; + owner.application = "myapp"; + owner.instance = "default"; return owner; } @@ -135,21 +174,29 @@ public class ChangeManagementAssessorTest { return membership; } - private NodeRepositoryNode createNode(String nodename, String hostname, String appName, String clusterId, int group) { + private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { + return createNode(nodename, hostname, clusterId, group, NodeType.tenant); + } + + private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group, NodeType nodeType) { NodeRepositoryNode node = new NodeRepositoryNode(); node.setHostname(nodename); node.setParentHostname(hostname); node.setState(NodeState.active); - node.setOwner(createOwner("mytenant", appName, "default")); + node.setOwner(createOwner()); node.setMembership(createMembership(clusterId, group)); + node.setType(nodeType); return node; } - private NodeRepositoryNode createHost(String hostname, String switchName) { + private NodeRepositoryNode createHost(String hostname, NodeType nodeType) { NodeRepositoryNode node = new NodeRepositoryNode(); node.setHostname(hostname); - node.setSwitchHostname(switchName); + node.setSwitchHostname("switch1"); + node.setType(nodeType); + node.setOwner(createOwner()); + node.setMembership(createMembership(nodeType.name(), 0)); return node; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index d718dc6b9cf..232521c9609 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -10,13 +10,15 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import org.junit.Test; import java.time.Duration; -import java.util.List; -import java.util.stream.Collectors; +import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; /** * @author bratseth @@ -27,12 +29,9 @@ public class DeploymentExpirerTest { @Test public void testDeploymentExpiry() { - tester.controllerTester().zoneRegistry().setDeploymentTimeToLive( - ZoneId.from(Environment.dev, RegionName.from("us-east-1")), - Duration.ofDays(14) - ); - DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(10) - ); + ZoneId devZone = ZoneId.from(Environment.dev, RegionName.from("us-east-1")); + tester.controllerTester().zoneRegistry().setDeploymentTimeToLive(devZone, Duration.ofDays(14)); + DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(1)); var devApp = tester.newDeploymentContext("tenant1", "app1", "default"); var prodApp = tester.newDeploymentContext("tenant2", "app2", "default"); @@ -45,27 +44,42 @@ public class DeploymentExpirerTest { // Deploy prod prodApp.submit(appPackage).deploy(); - - assertEquals(1, permanentDeployments(devApp.instance()).size()); - assertEquals(1, permanentDeployments(prodApp.instance()).size()); + assertEquals(1, permanentDeployments(devApp.instance())); + assertEquals(1, permanentDeployments(prodApp.instance())); // Not expired at first expirer.maintain(); - assertEquals(1, permanentDeployments(devApp.instance()).size()); - assertEquals(1, permanentDeployments(prodApp.instance()).size()); + assertEquals(1, permanentDeployments(devApp.instance())); + assertEquals(1, permanentDeployments(prodApp.instance())); + + // Deploy dev unsuccessfully a few days before expiry + tester.clock().advance(Duration.ofDays(12)); + tester.configServer().throwOnNextPrepare(new RuntimeException(getClass().getSimpleName())); + tester.jobs().deploy(devApp.instanceId(), JobType.devUsEast1, Optional.empty(), appPackage); + Run lastRun = tester.jobs().last(devApp.instanceId(), JobType.devUsEast1).get(); + assertSame(RunStatus.error, lastRun.status()); + Deployment deployment = tester.applications().requireInstance(devApp.instanceId()) + .deployments().get(devZone); + assertEquals("Time of last run is after time of deployment", Duration.ofDays(12), + Duration.between(deployment.at(), lastRun.end().get())); + + // Dev application does not expire based on time of successful deployment + tester.clock().advance(Duration.ofDays(2)); + expirer.maintain(); + assertEquals(1, permanentDeployments(devApp.instance())); + assertEquals(1, permanentDeployments(prodApp.instance())); - // The dev application is removed - tester.clock().advance(Duration.ofDays(15)); + // Dev application expires when enough time has passed since most recent attempt + tester.clock().advance(Duration.ofDays(12)); expirer.maintain(); - assertEquals(0, permanentDeployments(devApp.instance()).size()); - assertEquals(1, permanentDeployments(prodApp.instance()).size()); + assertEquals(0, permanentDeployments(devApp.instance())); + assertEquals(1, permanentDeployments(prodApp.instance())); } - private List<Deployment> permanentDeployments(Instance instance) { - return tester.controller().applications().getInstance(instance.id()).get().deployments().values().stream() - .filter(deployment -> deployment.zone().environment() != Environment.test && - deployment.zone().environment() != Environment.staging) - .collect(Collectors.toList()); + private long permanentDeployments(Instance instance) { + return tester.controller().applications().requireInstance(instance.id()).deployments().values().stream() + .filter(deployment -> !deployment.zone().environment().isTest()) + .count(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index e3830e274c9..1bc633122a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -37,12 +37,14 @@ public class OsUpgraderTest { public void upgrade_os() { CloudName cloud1 = CloudName.from("c1"); CloudName cloud2 = CloudName.from("c2"); + ZoneApi zone0 = zone("prod.us-north-42", "prod.controller", cloud1); ZoneApi zone1 = zone("prod.eu-west-1", cloud1); ZoneApi zone2 = zone("prod.us-west-1", cloud1); ZoneApi zone3 = zone("prod.us-central-1", cloud1); ZoneApi zone4 = zone("prod.us-east-3", cloud1); ZoneApi zone5 = zone("prod.us-north-1", cloud2); UpgradePolicy upgradePolicy = UpgradePolicy.create() + .upgrade(zone0) .upgrade(zone1) .upgradeInParallel(zone2, zone3) .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader @@ -52,8 +54,9 @@ public class OsUpgraderTest { // Bootstrap system tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.tenantHost)); + tester.configServer().addNodes(List.of(zone0.getVirtualId()), List.of(SystemApplication.controllerHost)); - // Add system applications that exist in a real system, but isn't upgraded + // Add system application that exists in a real system, but isn't eligible for OS upgrades tester.configServer().addNodes(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.configServer)); @@ -68,7 +71,15 @@ public class OsUpgraderTest { assertEquals(1, tester.controller().osVersionTargets().size()); // Only allows one version per cloud statusUpdater.maintain(); + // zone 0: controllers upgrade first + osUpgrader.maintain(); + assertWanted(version1, SystemApplication.controllerHost, zone0.getVirtualId()); + completeUpgrade(version1, SystemApplication.controllerHost, zone0.getVirtualId()); + statusUpdater.maintain(); + assertEquals(3, nodesOn(version1).size()); + // zone 1: begins upgrading + assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone1.getId()); osUpgrader.maintain(); assertWanted(version1, SystemApplication.tenantHost, zone1.getId()); @@ -78,7 +89,7 @@ public class OsUpgraderTest { // zone 1: completes upgrade completeUpgrade(version1, SystemApplication.tenantHost, zone1.getId()); statusUpdater.maintain(); - assertEquals(2, nodesOn(version1).size()); + assertEquals(5, nodesOn(version1).size()); assertEquals(11, nodesOn(Version.emptyVersion).size()); // zone 2 and 3: begins upgrading @@ -293,4 +304,8 @@ public class OsUpgraderTest { return ZoneApiMock.newBuilder().withId(id).with(cloud).build(); } + private static ZoneApi zone(String id, String virtualId, CloudName cloud) { + return ZoneApiMock.newBuilder().withId(id).withVirtualId(virtualId).with(cloud).build(); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 6370cfedc41..db2353860ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -179,10 +179,10 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), SystemApplication.all()); + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), SystemApplication.notController()); tester.upgradeSystem(version1); systemUpgrader.maintain(); - assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); + assertCurrentVersion(SystemApplication.notController(), version1, zone1, zone2, zone3, zone4); // Controller upgrades Version version2 = Version.fromString("6.6"); @@ -199,7 +199,7 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); completeUpgrade(SystemApplication.proxy, version2, zone1); convergeServices(SystemApplication.proxy, zone1); - assertWantedVersion(SystemApplication.all(), version1, zone2, zone3, zone4); + assertWantedVersion(SystemApplication.notController(), version1, zone2, zone3, zone4); // zone 2 and 3: systemUpgrader.maintain(); @@ -207,7 +207,7 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); completeUpgrade(SystemApplication.proxy, version2, zone2, zone3); convergeServices(SystemApplication.proxy, zone2, zone3); - assertWantedVersion(SystemApplication.all(), version1, zone4); + assertWantedVersion(SystemApplication.notController(), version1, zone4); // zone 4: systemUpgrader.maintain(); @@ -217,7 +217,7 @@ public class SystemUpgraderTest { // All done systemUpgrader.maintain(); - assertWantedVersion(SystemApplication.all(), version2, zone1, zone2, zone3, zone4); + assertWantedVersion(SystemApplication.notController(), version2, zone1, zone2, zone3, zone4); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 16a17d96c03..2dcf012ac6d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -95,7 +95,8 @@ public class ApplicationSerializerTest { .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b", Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); - deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils + deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3), + DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none)); deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), @@ -160,14 +161,8 @@ public class ApplicationSerializerTest { assertEquals(RotationStatus.EMPTY, serialized.require(id3.instance()).rotationStatus()); assertEquals(2, serialized.require(id1.instance()).deployments().size()); - assertEquals(original.require(id1.instance()).deployments().get(zone1).applicationVersion(), serialized.require(id1.instance()).deployments().get(zone1).applicationVersion()); - assertEquals(original.require(id1.instance()).deployments().get(zone2).applicationVersion(), serialized.require(id1.instance()).deployments().get(zone2).applicationVersion()); - assertEquals(original.require(id1.instance()).deployments().get(zone1).version(), serialized.require(id1.instance()).deployments().get(zone1).version()); - assertEquals(original.require(id1.instance()).deployments().get(zone2).version(), serialized.require(id1.instance()).deployments().get(zone2).version()); - assertEquals(original.require(id1.instance()).deployments().get(zone1).at(), serialized.require(id1.instance()).deployments().get(zone1).at()); - assertEquals(original.require(id1.instance()).deployments().get(zone2).at(), serialized.require(id1.instance()).deployments().get(zone2).at()); - assertEquals(original.require(id1.instance()).deployments().get(zone2).activity().lastQueried().get(), serialized.require(id1.instance()).deployments().get(zone2).activity().lastQueried().get()); - assertEquals(original.require(id1.instance()).deployments().get(zone2).activity().lastWritten().get(), serialized.require(id1.instance()).deployments().get(zone2).activity().lastWritten().get()); + assertEquals(original.require(id1.instance()).deployments().get(zone1), serialized.require(id1.instance()).deployments().get(zone1)); + assertEquals(original.require(id1.instance()).deployments().get(zone2), serialized.require(id1.instance()).deployments().get(zone2)); assertEquals(original.require(id1.instance()).jobPause(JobType.systemTest), serialized.require(id1.instance()).jobPause(JobType.systemTest)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json index 0cfb457660c..886a1dec5a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -12,9 +12,12 @@ "bandwidthGbps": 1.0, "diskSpeed": "slow", "storageType": "remote", - "fastDisk": false, "clusterId": "default", - "clusterType": "container" + "clusterType": "container", + "down": false, + "retired": false, + "restarting": false, + "rebooting": false } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java index 359f34f5740..2f2e70e2cf6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMemb import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.intellij.lang.annotations.Language; @@ -50,20 +51,20 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { private List<NodeRepositoryNode> createNodes() { List<NodeRepositoryNode> nodes = new ArrayList<>(); - nodes.add(createNode("node1", "host1", "myapp", "default", 0 )); - nodes.add(createNode("node2", "host1", "myapp", "default", 0 )); - nodes.add(createNode("node3", "host1", "myapp", "default", 0 )); - nodes.add(createNode("node4", "host2", "myapp", "default", 0 )); + nodes.add(createNode("node1", "host1", "default", 0 )); + nodes.add(createNode("node2", "host1", "default", 0 )); + nodes.add(createNode("node3", "host1", "default", 0 )); + nodes.add(createNode("node4", "host2", "default", 0 )); nodes.add(createHost("host1", "switch1")); nodes.add(createHost("host2", "switch2")); return nodes; } - private NodeOwner createOwner(String tenant, String application, String instance) { + private NodeOwner createOwner() { NodeOwner owner = new NodeOwner(); - owner.tenant = tenant; - owner.application = application; - owner.instance = instance; + owner.tenant = "mytenant"; + owner.application = "myapp"; + owner.instance = "default"; return owner; } @@ -77,13 +78,14 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { return membership; } - private NodeRepositoryNode createNode(String nodename, String hostname, String appName, String clusterId, int group) { + private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { NodeRepositoryNode node = new NodeRepositoryNode(); node.setHostname(nodename); node.setParentHostname(hostname); node.setState(NodeState.active); - node.setOwner(createOwner("mytenant", appName, "default")); + node.setOwner(createOwner()); node.setMembership(createMembership(clusterId, group)); + node.setType(NodeType.tenant); return node; } @@ -92,6 +94,9 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { NodeRepositoryNode node = new NodeRepositoryNode(); node.setHostname(hostname); node.setSwitchHostname(switchName); + node.setOwner(createOwner()); + node.setType(NodeType.host); + node.setMembership(createMembership("host", 0)); return node; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json index cfe98ab906b..cf349e06cff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json @@ -12,7 +12,7 @@ "groupsImpact": 1, "upgradePolicy": "na", "suggestedAction": "nothing", - "impact": "na" + "impact": "Impact larger than upgrade policy" } ], "hosts": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index f94dc7c562b..03e5eb2b7a8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -11,6 +11,12 @@ "user": "developer@tenant" }], "secretStores": [], + "integrations": { + "aws": { + "tenantRole": "my-tenant-tenant-role", + "accounts": [] + } + }, "quota": { "budget": null, "budgetUsed": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json index 25891755323..dc717b5cac0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json @@ -14,6 +14,18 @@ "role": "secret-role" } ], + "integrations": { + "aws": { + "tenantRole": "my-tenant-tenant-role", + "accounts": [ + { + "name": "secret-foo", + "awsId": "123", + "role": "secret-role" + } + ] + } + }, "quota": { "budget": null, "budgetUsed": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index 5965d4b5b00..14b900caf50 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -4,6 +4,12 @@ "creator": "administrator@tenant", "pemDeveloperKeys": [], "secretStores": [], + "integrations": { + "aws": { + "tenantRole": "my-tenant-tenant-role", + "accounts": [] + } + }, "quota": { "budget": null, "budgetUsed": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 38f5a60cf8a..e96af475216 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -584,7 +584,7 @@ public class RoutingPoliciesTest { .setZones(zones) .setRoutingMethod(zones, RoutingMethod.exclusive); tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone), - SystemApplication.all()); + SystemApplication.notController()); var context = tester.tester.newDeploymentContext(); var endpointId = EndpointId.of("r0"); @@ -750,7 +750,7 @@ public class RoutingPoliciesTest { tester.controllerTester().zoneRegistry().exclusiveRoutingIn(zones); } tester.controllerTester().configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), - SystemApplication.all()); + SystemApplication.notController()); } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java index dc2c001672d..454d93c7eab 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java @@ -34,9 +34,10 @@ public class StructReader { public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) { Field field = parent.getField(buffer.currentName()); - if (field == null) + if (field == null) { throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" + - parent.getDataType().getDataTypeName() + "'"); + parent.getDataType().getDataTypeName() + "', which has the fields:" + parent.getDataType().getFields()); + } return field; } diff --git a/eval/src/tests/instruction/fast_rename_optimizer/CMakeLists.txt b/eval/src/tests/instruction/fast_rename_optimizer/CMakeLists.txt index a69f26f6a85..3c4c3967b50 100644 --- a/eval/src/tests/instruction/fast_rename_optimizer/CMakeLists.txt +++ b/eval/src/tests/instruction/fast_rename_optimizer/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(eval_fast_rename_optimizer_test_app TEST fast_rename_optimizer_test.cpp DEPENDS vespaeval + GTest::GTest ) vespa_add_test(NAME eval_fast_rename_optimizer_test_app COMMAND eval_fast_rename_optimizer_test_app) diff --git a/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp b/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp index 3bc1472f2d5..58a3b119847 100644 --- a/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp +++ b/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp @@ -1,94 +1,80 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/instruction/replace_type_function.h> #include <vespa/eval/instruction/fast_rename_optimizer.h> #include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/eval_fixture.h> - +#include <vespa/eval/eval/test/value_compare.h> +#include <vespa/vespalib/util/unwind_message.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace vespalib; using namespace vespalib::eval; using namespace vespalib::eval::test; using namespace vespalib::eval::tensor_function; -const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); - -EvalFixture::ParamRepo make_params() { - return EvalFixture::ParamRepo() - .add("x5", GenSpec().idx("x", 5)) - .add("x5f", GenSpec().idx("x", 5).cells_float()) - .add("x_m", GenSpec().map("x", {"a", "b", "c"})) - .add("xy_mm", GenSpec().map("x", {"a", "b", "c"}).map("y", {"d","e"})) - .add("x5y3z_m", GenSpec().idx("x", 5).idx("y", 3).map("z", {"a","b"})) - .add("x5yz_m", GenSpec().idx("x", 5).map("y", {"a","b"}).map("z", {"d","e"})) - .add("x5y3", GenSpec().idx("x", 5).idx("y", 3)); -} -EvalFixture::ParamRepo param_repo = make_params(); +struct FunInfo { + using LookFor = ReplaceTypeFunction; + void verify(const LookFor &fun) const { + EXPECT_FALSE(fun.result_is_mutable()); + } +}; void verify_optimized(const vespalib::string &expr) { - EvalFixture fixture(prod_factory, expr, param_repo, true); - EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); - auto info = fixture.find_all<ReplaceTypeFunction>(); - EXPECT_EQUAL(info.size(), 1u); + CellTypeSpace all_types(CellTypeUtils::list_types(), 1); + EvalFixture::verify<FunInfo>(expr, {FunInfo{}}, all_types); } void verify_not_optimized(const vespalib::string &expr) { - EvalFixture fixture(prod_factory, expr, param_repo, true); - EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); - auto info = fixture.find_all<ReplaceTypeFunction>(); - EXPECT_TRUE(info.empty()); -} - -TEST("require that non-transposing dense renames are optimized") { - TEST_DO(verify_optimized("rename(x5,x,y)")); - TEST_DO(verify_optimized("rename(x5,x,a)")); - TEST_DO(verify_optimized("rename(x5y3,y,z)")); - TEST_DO(verify_optimized("rename(x5y3,x,a)")); - TEST_DO(verify_optimized("rename(x5y3,(x,y),(a,b))")); - TEST_DO(verify_optimized("rename(x5y3,(x,y),(z,zz))")); - TEST_DO(verify_optimized("rename(x5y3,(x,y),(y,z))")); - TEST_DO(verify_optimized("rename(x5y3,(y,x),(b,a))")); + CellTypeSpace all_types(CellTypeUtils::list_types(), 1); + EvalFixture::verify<FunInfo>(expr, {}, all_types); } -TEST("require that transposing dense renames are not optimized") { - TEST_DO(verify_not_optimized("rename(x5y3,x,z)")); - TEST_DO(verify_not_optimized("rename(x5y3,y,a)")); - TEST_DO(verify_not_optimized("rename(x5y3,(x,y),(y,x))")); - TEST_DO(verify_not_optimized("rename(x5y3,(x,y),(b,a))")); - TEST_DO(verify_not_optimized("rename(x5y3,(y,x),(a,b))")); +TEST(FastRenameTest, non_transposing_dense_renames_are_optimized) { + UNWIND_DO(verify_optimized("rename(x5,x,y)")); + UNWIND_DO(verify_optimized("rename(x5,x,a)")); + UNWIND_DO(verify_optimized("rename(x5y3,y,z)")); + UNWIND_DO(verify_optimized("rename(x5y3,x,a)")); + UNWIND_DO(verify_optimized("rename(x5y3,(x,y),(a,b))")); + UNWIND_DO(verify_optimized("rename(x5y3,(x,y),(z,zz))")); + UNWIND_DO(verify_optimized("rename(x5y3,(x,y),(y,z))")); + UNWIND_DO(verify_optimized("rename(x5y3,(y,x),(b,a))")); } -TEST("require that non-dense renames may be optimized") { - TEST_DO(verify_optimized("rename(x_m,x,y)")); - TEST_DO(verify_optimized("rename(xy_mm,(x,y),(a,b))")); - TEST_DO(verify_optimized("rename(xy_mm,(x,y),(y,z))")); - TEST_DO(verify_not_optimized("rename(xy_mm,(x,y),(b,a))")); - TEST_DO(verify_not_optimized("rename(xy_mm,(x,y),(y,x))")); - - TEST_DO(verify_optimized("rename(x5y3z_m,(z),(a))")); - TEST_DO(verify_optimized("rename(x5y3z_m,(x,y,z),(b,c,a))")); - TEST_DO(verify_optimized("rename(x5y3z_m,(z),(a))")); - TEST_DO(verify_optimized("rename(x5y3z_m,(x,y,z),(b,c,a))")); - TEST_DO(verify_not_optimized("rename(x5y3z_m,(y),(a))")); - TEST_DO(verify_not_optimized("rename(x5y3z_m,(x,z),(z,x))")); - - TEST_DO(verify_optimized("rename(x5yz_m,(x,y),(y,x))")); - TEST_DO(verify_optimized("rename(x5yz_m,(x,y,z),(c,a,b))")); - TEST_DO(verify_optimized("rename(x5yz_m,(y,z),(a,b))")); - TEST_DO(verify_not_optimized("rename(x5yz_m,(z),(a))")); - TEST_DO(verify_not_optimized("rename(x5yz_m,(y,z),(z,y))")); +TEST(FastRenameTest, transposing_dense_renames_are_not_optimized) { + UNWIND_DO(verify_not_optimized("rename(x5y3,x,z)")); + UNWIND_DO(verify_not_optimized("rename(x5y3,y,a)")); + UNWIND_DO(verify_not_optimized("rename(x5y3,(x,y),(y,x))")); + UNWIND_DO(verify_not_optimized("rename(x5y3,(x,y),(b,a))")); + UNWIND_DO(verify_not_optimized("rename(x5y3,(y,x),(a,b))")); } -TEST("require that chained optimized renames are compacted into a single operation") { - TEST_DO(verify_optimized("rename(rename(x5,x,y),y,z)")); +TEST(FastRenameTest, non_dense_renames_may_be_optimized) { + UNWIND_DO(verify_optimized("rename(x3_1,x,y)")); + UNWIND_DO(verify_optimized("rename(x3_1y2_1,(x,y),(a,b))")); + UNWIND_DO(verify_optimized("rename(x3_1y2_1,(x,y),(y,z))")); + UNWIND_DO(verify_not_optimized("rename(x3_1y2_1,(x,y),(b,a))")); + UNWIND_DO(verify_not_optimized("rename(x3_1y2_1,(x,y),(y,x))")); + + UNWIND_DO(verify_optimized("rename(x5y3z2_1,(z),(a))")); + UNWIND_DO(verify_optimized("rename(x5y3z2_1,(x,y,z),(b,c,a))")); + UNWIND_DO(verify_optimized("rename(x5y3z2_1,(z),(a))")); + UNWIND_DO(verify_optimized("rename(x5y3z2_1,(x,y,z),(b,c,a))")); + UNWIND_DO(verify_not_optimized("rename(x5y3z2_1,(y),(a))")); + UNWIND_DO(verify_not_optimized("rename(x5y3z2_1,(x,z),(z,x))")); + + UNWIND_DO(verify_optimized("rename(x5y2_1z9_3,(x,y),(y,x))")); + UNWIND_DO(verify_optimized("rename(x5y2_1z9_3,(x,y,z),(c,a,b))")); + UNWIND_DO(verify_optimized("rename(x5y2_1z9_3,(y,z),(a,b))")); + UNWIND_DO(verify_not_optimized("rename(x5y2_1z9_3,(z),(a))")); + UNWIND_DO(verify_not_optimized("rename(x5y2_1z9_3,(y,z),(z,y))")); } -TEST("require that optimization works for float cells") { - TEST_DO(verify_optimized("rename(x5f,x,y)")); +TEST(FastRenameTest, chained_optimized_renames_are_compacted_into_a_single_operation) { + UNWIND_DO(verify_optimized("rename(rename(x5,x,y),y,z)")); } bool is_stable(const vespalib::string &from_spec, const vespalib::string &to_spec, @@ -99,31 +85,31 @@ bool is_stable(const vespalib::string &from_spec, const vespalib::string &to_spe return FastRenameOptimizer::is_stable_rename(from_type, to_type, from, to); } -TEST("require that rename is stable if dimension order is preserved") { +TEST(FastRenameTest, rename_is_stable_if_dimension_order_is_preserved) { EXPECT_TRUE(is_stable("tensor(a{},b{})", "tensor(a{},c{})", {"b"}, {"c"})); EXPECT_TRUE(is_stable("tensor(c[3],d[5])", "tensor(c[3],e[5])", {"d"}, {"e"})); EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(a{},b{},c[3],e[5])", {"d"}, {"e"})); EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(e{},f{},g[3],h[5])", {"a", "b", "c", "d"}, {"e", "f", "g", "h"})); } -TEST("require that rename is unstable if nontrivial indexed dimensions change order") { +TEST(FastRenameTest, rename_is_unstable_if_nontrivial_indexed_dimensions_change_order) { EXPECT_FALSE(is_stable("tensor(c[3],d[5])", "tensor(d[5],e[3])", {"c"}, {"e"})); EXPECT_FALSE(is_stable("tensor(c[3],d[5])", "tensor(c[5],d[3])", {"c", "d"}, {"d", "c"})); } -TEST("require that rename is unstable if mapped dimensions change order") { +TEST(FastRenameTest, rename_is_unstable_if_mapped_dimensions_change_order) { EXPECT_FALSE(is_stable("tensor(a{},b{})", "tensor(b{},c{})", {"a"}, {"c"})); EXPECT_FALSE(is_stable("tensor(a{},b{})", "tensor(a{},b{})", {"a", "b"}, {"b", "a"})); } -TEST("require that rename can be stable if indexed and mapped dimensions change order") { +TEST(FastRenameTest, rename_can_be_stable_if_indexed_and_mapped_dimensions_change_order) { EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(a[3],b[5],c{},d{})", {"a", "b", "c", "d"}, {"c", "d", "a", "b"})); EXPECT_TRUE(is_stable("tensor(a{},b{},c[3],d[5])", "tensor(c[3],d[5],e{},f{})", {"a", "b"}, {"e", "f"})); } -TEST("require that rename can be stable if trivial dimension is moved") { +TEST(FastRenameTest, rename_can_be_stable_if_trivial_dimension_is_moved) { EXPECT_TRUE(is_stable("tensor(a[1],b{},c[3])", "tensor(b{},bb[1],c[3])", {"a"}, {"bb"})); EXPECT_TRUE(is_stable("tensor(a[1],b{},c[3])", "tensor(b{},c[3],cc[1])", {"a"}, {"cc"})); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp b/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp index 95d3d882f7b..839c7bcb64a 100644 --- a/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp +++ b/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp @@ -30,21 +30,21 @@ struct FunInfo { }; void assert_mixed_optimized(const vespalib::string &expr) { - TEST_STATE(expr.c_str()); + SCOPED_TRACE(expr.c_str()); CellTypeSpace all_types(CellTypeUtils::list_types(), 2); using MIP = FunInfo<MixedInnerProductFunction>; EvalFixture::verify<MIP>(expr, {MIP{}}, all_types); } void assert_not_mixed_optimized(const vespalib::string &expr) { - TEST_STATE(expr.c_str()); + SCOPED_TRACE(expr.c_str()); CellTypeSpace all_types(CellTypeUtils::list_types(), 2); using MIP = FunInfo<MixedInnerProductFunction>; EvalFixture::verify<MIP>(expr, {}, all_types); } void assert_dense_optimized(const vespalib::string &expr) { - TEST_STATE(expr.c_str()); + SCOPED_TRACE(expr.c_str()); CellTypeSpace all_types(CellTypeUtils::list_types(), 2); using MIP = FunInfo<MixedInnerProductFunction>; EvalFixture::verify<MIP>(expr, {}, all_types); diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index 77cc1231f9c..47a121d7750 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -19,7 +19,7 @@ std::shared_ptr<Function const> verify_function(std::shared_ptr<Function const> if (fun->has_error()) { fprintf(stderr, "eval_fixture: function parse failed: %s\n", fun->get_error().c_str()); } - ASSERT_TRUE(!fun->has_error()); + REQUIRE(!fun->has_error()); return fun; } @@ -28,11 +28,11 @@ NodeTypes get_types(const Function &function, const ParamRepo ¶m_repo) { for (size_t i = 0; i < function.num_params(); ++i) { auto pos = param_repo.map.find(function.param_name(i)); if (pos == param_repo.map.end()) { - TEST_STATE(fmt("param name: '%s'", function.param_name(i).data()).c_str()); - ASSERT_TRUE(pos != param_repo.map.end()); + UNWIND_MSG("param name: '%s'", function.param_name(i).data()); + REQUIRE(pos != param_repo.map.end()); } param_types.push_back(ValueType::from_spec(pos->second.value.type())); - ASSERT_TRUE(!param_types.back().is_error()); + REQUIRE(!param_types.back().is_error()); } NodeTypes node_types(function, param_types); if (!node_types.errors().empty()) { @@ -40,7 +40,7 @@ NodeTypes get_types(const Function &function, const ParamRepo ¶m_repo) { fprintf(stderr, "eval_fixture: type error: %s\n", msg.c_str()); } } - ASSERT_TRUE(node_types.errors().empty()); + REQUIRE(node_types.errors().empty()); return node_types; } @@ -48,7 +48,7 @@ std::set<size_t> get_mutable(const Function &function, const ParamRepo ¶m_re std::set<size_t> mutable_set; for (size_t i = 0; i < function.num_params(); ++i) { auto pos = param_repo.map.find(function.param_name(i)); - ASSERT_TRUE(pos != param_repo.map.end()); + REQUIRE(pos != param_repo.map.end()); if (pos->second.is_mutable) { mutable_set.insert(i); } @@ -90,7 +90,7 @@ std::vector<Value::UP> make_params(const ValueBuilderFactory &factory, const Fun std::vector<Value::UP> result; for (size_t i = 0; i < function.num_params(); ++i) { auto pos = param_repo.map.find(function.param_name(i)); - ASSERT_TRUE(pos != param_repo.map.end()); + REQUIRE(pos != param_repo.map.end()); result.push_back(value_from_spec(pos->second.value, factory)); } return result; @@ -109,7 +109,7 @@ std::vector<Value::CREF> get_refs(const std::vector<Value::UP> &values) { ParamRepo & EvalFixture::ParamRepo::add(const vespalib::string &name, TensorSpec value) { - ASSERT_TRUE(map.find(name) == map.end()); + REQUIRE(map.find(name) == map.end()); map.insert_or_assign(name, Param(std::move(value), false)); return *this; } @@ -117,7 +117,7 @@ EvalFixture::ParamRepo::add(const vespalib::string &name, TensorSpec value) ParamRepo & EvalFixture::ParamRepo::add_mutable(const vespalib::string &name, TensorSpec value) { - ASSERT_TRUE(map.find(name) == map.end()); + REQUIRE(map.find(name) == map.end()); map.insert_or_assign(name, Param(std::move(value), true)); return *this; } @@ -166,10 +166,10 @@ EvalFixture::detect_param_tampering(const ParamRepo ¶m_repo, bool allow_muta { for (size_t i = 0; i < _function->num_params(); ++i) { auto pos = param_repo.map.find(_function->param_name(i)); - ASSERT_TRUE(pos != param_repo.map.end()); + REQUIRE(pos != param_repo.map.end()); bool allow_tampering = allow_mutable && pos->second.is_mutable; if (!allow_tampering) { - ASSERT_EQUAL(pos->second.value, spec_from_value(*_param_values[i])); + REQUIRE_EQ(pos->second.value, spec_from_value(*_param_values[i])); } } } @@ -195,8 +195,8 @@ EvalFixture::EvalFixture(const ValueBuilderFactory &factory, _result(spec_from_value(_result_value)) { auto result_type = ValueType::from_spec(_result.type()); - ASSERT_TRUE(!result_type.is_error()); - TEST_DO(detect_param_tampering(param_repo, allow_mutable)); + REQUIRE(!result_type.is_error()); + UNWIND_DO(detect_param_tampering(param_repo, allow_mutable)); } size_t @@ -212,7 +212,7 @@ EvalFixture::ref(const vespalib::string &expr, const ParamRepo ¶m_repo) std::vector<TensorSpec> params; for (size_t i = 0; i < fun->num_params(); ++i) { auto pos = param_repo.map.find(fun->param_name(i)); - ASSERT_TRUE(pos != param_repo.map.end()); + REQUIRE(pos != param_repo.map.end()); params.push_back(pos->second.value); } return ReferenceEvaluation::eval(*fun, params); diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index e99e6755317..c726fd4af94 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -13,7 +13,8 @@ #include <functional> #include "gen_spec.h" #include "cell_type_space.h" -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/require.h> +#include <vespa/vespalib/util/unwind_message.h> namespace vespalib::eval::test { @@ -140,8 +141,10 @@ public: template <typename FunInfo> static void verify(const vespalib::string &expr, const std::vector<FunInfo> &fun_info, CellTypeSpace cell_type_space) { + + UNWIND_MSG("in verify(%s) with %zu FunInfo", expr.c_str(), fun_info.size()); auto fun = Function::parse(expr); - ASSERT_EQUAL(fun->num_params(), cell_type_space.n()); + REQUIRE_EQ(fun->num_params(), cell_type_space.n()); for (; cell_type_space.valid(); cell_type_space.next()) { auto cell_types = cell_type_space.get(); EvalFixture::ParamRepo param_repo; @@ -151,11 +154,11 @@ public: EvalFixture fixture(prod_factory(), expr, param_repo, true, true); EvalFixture slow_fixture(prod_factory(), expr, param_repo, false, false); EvalFixture test_fixture(test_factory(), expr, param_repo, true, true); - ASSERT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); - ASSERT_EQUAL(fixture.result(), slow_fixture.result()); - ASSERT_EQUAL(fixture.result(), test_fixture.result()); + REQUIRE_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + REQUIRE_EQ(fixture.result(), slow_fixture.result()); + REQUIRE_EQ(fixture.result(), test_fixture.result()); auto info = fixture.find_all<typename FunInfo::LookFor>(); - ASSERT_EQUAL(info.size(), fun_info.size()); + REQUIRE_EQ(info.size(), fun_info.size()); for (size_t i = 0; i < fun_info.size(); ++i) { fixture.verify_callback<FunInfo>(fun_info[i], *info[i]); } diff --git a/eval/src/vespa/eval/eval/test/gen_spec.cpp b/eval/src/vespa/eval/eval/test/gen_spec.cpp index 96c2c16eaa0..0b624a457d7 100644 --- a/eval/src/vespa/eval/eval/test/gen_spec.cpp +++ b/eval/src/vespa/eval/eval/test/gen_spec.cpp @@ -2,6 +2,7 @@ #include "gen_spec.h" #include <vespa/eval/eval/string_stuff.h> +#include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/stringfmt.h> using vespalib::make_string_short::fmt; @@ -35,7 +36,7 @@ Sequence SigmoidF(const Sequence &seq) { } Sequence Seq(const std::vector<double> &seq) { - assert(!seq.empty()); + REQUIRE(!seq.empty()); return [seq](size_t i) noexcept { return seq[i % seq.size()]; }; } @@ -70,23 +71,23 @@ DimSpec::from_desc(const vespalib::string &desc) auto as_num = [](char c) { return size_t(c - '0'); }; auto is_map_tag = [](char c) { return (c == '_'); }; auto extract_number = [&]() { - assert(idx < desc.size()); - assert(is_num(desc[idx])); + REQUIRE(idx < desc.size()); + REQUIRE(is_num(desc[idx])); size_t num = as_num(desc[idx++]); while ((idx < desc.size()) && is_num(desc[idx])) { num = (num * 10) + as_num(desc[idx++]); } return num; }; - assert(!desc.empty()); - assert(is_dim_name(desc[idx])); + REQUIRE(!desc.empty()); + REQUIRE(is_dim_name(desc[idx])); name.push_back(desc[idx++]); size_t size = extract_number(); if (idx < desc.size()) { // mapped - assert(is_map_tag(desc[idx++])); + REQUIRE(is_map_tag(desc[idx++])); size_t stride = extract_number(); - assert(idx == desc.size()); + REQUIRE(idx == desc.size()); return {name, make_dict(size, stride, "")}; } else { // indexed @@ -104,7 +105,7 @@ GenSpec::from_desc(const vespalib::string &desc) std::vector<DimSpec> dim_list; while (idx < desc.size()) { dim_desc.clear(); - assert(is_dim_name(desc[idx])); + REQUIRE(is_dim_name(desc[idx])); dim_desc.push_back(desc[idx++]); while ((idx < desc.size()) && !is_dim_name(desc[idx])) { dim_desc.push_back(desc[idx++]); @@ -135,7 +136,7 @@ GenSpec::type() const dim_types.push_back(dim.type()); } auto type = ValueType::make_type(_cells, dim_types); - assert(!type.is_error()); + REQUIRE(!type.is_error()); return type; } @@ -144,7 +145,7 @@ GenSpec::gen() const { size_t idx = 0; TensorSpec::Address addr; - assert(!bad_scalar()); + REQUIRE(!bad_scalar()); TensorSpec result(type().to_spec()); std::function<void(size_t)> add_cells = [&](size_t dim_idx) { if (dim_idx == _dims.size()) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b583eaffc7e..50419577e7a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -202,7 +202,7 @@ public class Flags { public static final UnboundBooleanFlag GROUP_SUSPENSION = defineFeatureFlag( "group-suspension", true, - List.of("hakon"), "2021-01-22", "2021-03-22", + List.of("hakon"), "2021-01-22", "2021-04-22", "Allow all content nodes in a hierarchical group to suspend at the same time", "Takes effect on the next suspension request to the Orchestrator.", APPLICATION_ID); diff --git a/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/VespaHttpClientBuilder.java index 741570e950b..53bf7b866af 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/VespaHttpClientBuilder.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http; +package ai.vespa.util.http.hc4; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TlsContext; diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelaySupplier.java index a97024df95c..b202966c412 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelaySupplier.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import java.time.Duration; diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandler.java index dca1907c08b..3ba92c08e30 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandler.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandler.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import org.apache.http.annotation.Contract; import org.apache.http.annotation.ThreadingBehavior; diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandler.java index 041b501809c..d4ceb44a3ab 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandler.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import org.apache.http.HttpResponse; import org.apache.http.annotation.Contract; diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryConsumer.java index 494be051673..c168f7d50c9 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryConsumer.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import org.apache.http.client.protocol.HttpClientContext; diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryFailedConsumer.java index ed326ac1210..801c8a5af2f 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryFailedConsumer.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import org.apache.http.client.protocol.HttpClientContext; diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryPredicate.java index ccf62c9be9f..45c5ef0d623 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/RetryPredicate.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import org.apache.http.client.protocol.HttpClientContext; diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/Sleeper.java index 25f5b9eb627..f593561888d 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc4/retry/Sleeper.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import java.time.Duration; diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java new file mode 100644 index 00000000000..672a2fd3918 --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.hc5; + +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.client5.http.impl.DefaultSchemePortResolver; +import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.client5.http.routing.HttpRoutePlanner; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.protocol.HttpContext; + +/** + * {@link HttpRoutePlanner} that changes assumes requests specify the HTTP scheme, + * and then changes this to HTTPS, keeping the other host parameters. + * + * @author jonmv + */ +class HttpToHttpsRoutePlanner implements HttpRoutePlanner { + + @Override + public HttpRoute determineRoute(HttpHost target, HttpContext context) throws HttpException { + if ( ! target.getSchemeName().equals("http")) + throw new IllegalArgumentException("Scheme must be 'http' when using HttpToHttpsRoutePlanner"); + + if (target.getPort() == -1) + throw new IllegalArgumentException("Port must be set when using HttpToHttpsRoutePlanner"); + + if (HttpClientContext.adapt(context).getRequestConfig().getProxy() != null) + throw new IllegalArgumentException("Proxies are not supported with HttpToHttpsRoutePlanner"); + + return new HttpRoute(new HttpHost("https", target.getAddress(), target.getHostName(), target.getPort())); + } + +} diff --git a/http-utils/src/main/java/ai/vespa/util/http/VespaAsyncHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaAsyncHttpClientBuilder.java index f5457e17e96..219f1707589 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/VespaAsyncHttpClientBuilder.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaAsyncHttpClientBuilder.java @@ -1,22 +1,15 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http; +package ai.vespa.util.http.hc5; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; -import org.apache.hc.client5.http.HttpRoute; -import org.apache.hc.client5.http.impl.DefaultSchemePortResolver; import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; -import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner; import org.apache.hc.client5.http.nio.AsyncClientConnectionManager; -import org.apache.hc.client5.http.routing.HttpRoutePlanner; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; -import org.apache.hc.core5.http.HttpException; -import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; -import org.apache.hc.core5.http.protocol.HttpContext; import javax.net.ssl.SSLParameters; @@ -70,27 +63,4 @@ public class VespaAsyncHttpClientBuilder { return clientBuilder; } - private static class HttpToHttpsRoutePlanner implements HttpRoutePlanner { - - private final DefaultRoutePlanner defaultPlanner = new DefaultRoutePlanner(new DefaultSchemePortResolver()); - - @Override - public HttpRoute determineRoute(HttpHost target, HttpContext context) throws HttpException { - HttpRoute originalRoute = defaultPlanner.determineRoute(target, context); - HttpHost originalHost = originalRoute.getTargetHost(); - String originalScheme = originalHost.getSchemeName(); - String rewrittenScheme = originalScheme.equalsIgnoreCase("http") ? "https" : originalScheme; - boolean rewrittenSecure = target.getSchemeName().equalsIgnoreCase("https"); - HttpHost rewrittenHost = new HttpHost( - rewrittenScheme, originalHost.getAddress(), originalHost.getHostName(), originalHost.getPort()); - return new HttpRoute( - rewrittenHost, - originalRoute.getLocalAddress(), - originalRoute.getProxyHost(), - rewrittenSecure, - originalRoute.getTunnelType(), - originalRoute.getLayerType()); - } - } - } diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java new file mode 100644 index 00000000000..2824a1b801d --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/VespaHttpClientBuilder.java @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.hc5; + +import com.yahoo.security.tls.TransportSecurityUtils; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; +import org.apache.hc.client5.http.socket.ConnectionSocketFactory; +import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; +import org.apache.hc.core5.http.config.Registry; +import org.apache.hc.core5.http.config.RegistryBuilder; + +import javax.net.ssl.SSLParameters; + +import static com.yahoo.security.tls.MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER; +import static com.yahoo.security.tls.TransportSecurityUtils.getInsecureMixedMode; +import static com.yahoo.security.tls.TransportSecurityUtils.getSystemTlsContext; +import static com.yahoo.security.tls.TransportSecurityUtils.isTransportSecurityEnabled; + +/** + * Sync HTTP client builder <em>for internal Vespa communications over http/https.</em> + * + * Configures Vespa mTLS and handles TLS mixed mode automatically. + * Custom connection managers must be configured through {@link #create(HttpClientConnectionManagerFactory)}. + * + * @author jonmv + */ +public class VespaHttpClientBuilder { + + public interface HttpClientConnectionManagerFactory { + HttpClientConnectionManager create(Registry<ConnectionSocketFactory> socketFactories); + } + + public static HttpClientBuilder create() { + return create(PoolingHttpClientConnectionManager::new); + } + + public static HttpClientBuilder create(HttpClientConnectionManagerFactory connectionManagerFactory) { + HttpClientBuilder builder = HttpClientBuilder.create(); + addSslSocketFactory(builder, connectionManagerFactory); + addHttpsRewritingRoutePlanner(builder); + + builder.disableConnectionState(); // Share connections between subsequent requests. + builder.disableCookieManagement(); + builder.disableAuthCaching(); + builder.disableRedirectHandling(); + + return builder; + } + + private static void addSslSocketFactory(HttpClientBuilder builder, HttpClientConnectionManagerFactory connectionManagerFactory) { + getSystemTlsContext().ifPresent(tlsContext -> { + SSLParameters parameters = tlsContext.parameters(); + SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory(tlsContext.context(), + parameters.getProtocols(), + parameters.getCipherSuites(), + new NoopHostnameVerifier()); + builder.setConnectionManager(connectionManagerFactory.create(createRegistry(socketFactory))); + // Workaround that allows re-using https connections, see https://stackoverflow.com/a/42112034/1615280 for details. + // Proper solution would be to add a request interceptor that adds a x500 principal as user token, + // but certificate subject CN is not accessible through the TlsContext currently. + builder.setUserTokenHandler((route, context) -> null); + }); + } + + private static Registry<ConnectionSocketFactory> createRegistry(SSLConnectionSocketFactory sslSocketFactory) { + return RegistryBuilder.<ConnectionSocketFactory>create() + .register("https", sslSocketFactory) + .register("http", PlainConnectionSocketFactory.getSocketFactory()) + .build(); + } + + private static void addHttpsRewritingRoutePlanner(HttpClientBuilder builder) { + if (isTransportSecurityEnabled() && getInsecureMixedMode() != PLAINTEXT_CLIENT_MIXED_SERVER) + builder.setRoutePlanner(new HttpToHttpsRoutePlanner()); + } + +} diff --git a/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/VespaHttpClientBuilderTest.java index b9a8fd748d6..58aa70b69b1 100644 --- a/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/VespaHttpClientBuilderTest.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http; +package ai.vespa.util.http.hc4; import org.apache.http.HttpException; import org.apache.http.HttpHost; diff --git a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java index 82bf4ff8080..7330a91d75c 100644 --- a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandlerTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import org.apache.http.client.protocol.HttpClientContext; import org.junit.Test; diff --git a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java index 258f29f4ced..514eae56fe8 100644 --- a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java @@ -1,5 +1,5 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.util.http.retry; +package ai.vespa.util.http.hc4.retry; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; diff --git a/http-utils/src/test/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlannerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlannerTest.java new file mode 100644 index 00000000000..58dc25fdf1a --- /dev/null +++ b/http-utils/src/test/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlannerTest.java @@ -0,0 +1,59 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.hc5; + +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHost; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class HttpToHttpsRoutePlannerTest { + + final HttpToHttpsRoutePlanner planner = new HttpToHttpsRoutePlanner(); + + @Test + public void verifySchemeMustBeHttp() throws HttpException { + try { + planner.determineRoute(new HttpHost("https", "host", 1), new HttpClientContext()); + } + catch (IllegalArgumentException e) { + assertEquals("Scheme must be 'http' when using HttpToHttpsRoutePlanner", e.getMessage()); + } + } + + @Test + public void verifyPortMustBeSet() throws HttpException { + try { + planner.determineRoute(new HttpHost("http", "host", -1), new HttpClientContext()); + } + catch (IllegalArgumentException e) { + assertEquals("Port must be set when using HttpToHttpsRoutePlanner", e.getMessage()); + } + } + + + @Test + public void verifyProxyIsDisallowed() throws HttpException { + HttpClientContext context = new HttpClientContext(); + context.setRequestConfig(RequestConfig.custom().setProxy(new HttpHost("proxy")).build()); + try { + planner.determineRoute(new HttpHost("http", "host", 1), context); + } + catch (IllegalArgumentException e) { + assertEquals("Proxies are not supported with HttpToHttpsRoutePlanner", e.getMessage()); + } + } + + @Test + public void verifySchemeIsRewritten() throws HttpException { + assertEquals(new HttpRoute(new HttpHost("https", "host", 1)), + planner.determineRoute(new HttpHost("http", "host", 1), new HttpClientContext())); + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index 15f924505be..bef4864fb6d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -3,7 +3,7 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import java.util.logging.Level; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index d1e1d6f14cc..05a2b85af68 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -1,7 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import org.apache.http.client.config.RequestConfig; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 314d556b9b4..72f77926099 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -22,14 +22,11 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { * Connect to remote service over http and fetch metrics */ public Metrics getMetrics(int fetchCount) { - String data = "{}"; try { - data = getJson(); + return createMetrics(getJson(), fetchCount); } catch (IOException e) { - logMessageNoResponse(errMsgNoResponse(e), fetchCount); + return new Metrics(); } - - return createMetrics(data, fetchCount); } Metrics createMetrics(String data, int fetchCount) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 2d9b61b6ceb..3133a5568f7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -617,10 +617,8 @@ public class NodeAgentImpl implements NodeAgent { Optional<NodeMembership> membership = context.node().membership(); return zone.getSystemName().isCd() || zone.getEnvironment().isTest() - || (context.nodeType() != NodeType.tenant) - || membership.map(mem -> ! (mem.type().isContainer() || - mem.type().isCombined())) - .orElse(false) + || context.nodeType() != NodeType.tenant + || membership.map(mem -> ! (mem.type().isContainer() || mem.type().isAdmin())).orElse(false) ? Duration.ofSeconds(-1) : warmUpDuration; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index bba4e93616e..197193fafa9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -33,6 +33,11 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { super(nodes, negate, NodeList::new); } + /** Returns the node with the given hostname from this list, or empty if it is not present */ + public Optional<Node> node(String hostname) { + return matching(node -> node.hostname().equals(hostname)).first(); + } + /** Returns the subset of nodes which are retired */ public NodeList retired() { return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().retired()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java new file mode 100644 index 00000000000..ca18028ad5a --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -0,0 +1,152 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision; + +import com.yahoo.collections.Pair; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.autoscale.Load; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; +import com.yahoo.vespa.hosted.provision.autoscale.NodeTimeseries; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Stats about the current state known to this node repo + * + * @author bratseth + */ +public class NodeRepoStats { + + private final Load load; + private final Load activeLoad; + private final List<ApplicationStats> applicationStats; + + private NodeRepoStats(Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + this.load = load; + this.activeLoad = activeLoad; + this.applicationStats = List.copyOf(applicationStats); + } + + /** + * Returns the current average work-extracting utilization in this node repo over all nodes. + * Capacity not allocated to active nodes are taken to have 0 utilization as it provides no useful work. + */ + public Load load() { return load; } + + /** Returns the current average utilization in this node repo over all active nodes. */ + public Load activeLoad() { return activeLoad; } + + /** Returns stats on each application, sorted by decreasing unutilized allocation measured in dollars per hour. */ + public List<ApplicationStats> applicationStats() { return applicationStats; } + + public static NodeRepoStats computeOver(NodeRepository nodeRepository) { + NodeList allNodes = nodeRepository.nodes().list(); + List<NodeTimeseries> allNodeTimeseries = nodeRepository.metricsDb().getNodeTimeseries(Duration.ofHours(1), Set.of()); + + Pair<Load, Load> load = computeLoad(allNodes, allNodeTimeseries); + List<ApplicationStats> applicationStats = computeApplicationStats(allNodes, allNodeTimeseries); + return new NodeRepoStats(load.getFirst(), load.getSecond(), applicationStats); + } + + private static Pair<Load, Load> computeLoad(NodeList allNodes, List<NodeTimeseries> allNodeTimeseries) { + NodeResources totalActiveResources = NodeResources.zero(); + Load load = Load.zero(); + for (var nodeTimeseries : allNodeTimeseries) { + Optional<Node> node = allNodes.node(nodeTimeseries.hostname()); + if (node.isEmpty() || node.get().state() != Node.State.active) continue; + + Optional<NodeMetricSnapshot> snapshot = nodeTimeseries.last(); + if (snapshot.isEmpty()) continue; + + load = load.add(snapshot.get().load().multiply(node.get().resources())); + totalActiveResources = totalActiveResources.add(node.get().resources().justNumbers()); + } + + NodeResources totalHostResources = NodeResources.zero(); + for (var host : allNodes.hosts()) + totalHostResources = totalHostResources.add(host.resources().justNumbers()); + + return new Pair<>(load.divide(totalHostResources), load.divide(totalActiveResources)); + } + + private static List<ApplicationStats> computeApplicationStats(NodeList allNodes, + List<NodeTimeseries> allNodeTimeseries) { + List<ApplicationStats> applicationStats = new ArrayList<>(); + Map<String, NodeMetricSnapshot> snapshotsByHost = byHost(allNodeTimeseries); + for (var applicationNodes : allNodes.state(Node.State.active) + .nodeType(NodeType.tenant) + .matching(node -> ! node.allocation().get().owner().instance().isTester()) + .groupingBy(node -> node.allocation().get().owner()).entrySet()) { + + NodeResources totalResources = NodeResources.zero(); + NodeResources totalUtilizedResources = NodeResources.zero(); + for (var node : applicationNodes.getValue()) { + var snapshot = snapshotsByHost.get(node.hostname()); + if (snapshot == null) continue; + + totalResources = totalResources.add(node.resources().justNumbers()); + totalUtilizedResources = totalUtilizedResources.add(snapshot.load().scaled(node.resources().justNumbers())); + } + applicationStats.add(new ApplicationStats(applicationNodes.getKey(), + Load.byDividing(totalUtilizedResources, totalResources), + totalResources.cost(), + totalUtilizedResources.cost())); + } + Collections.sort(applicationStats); + return applicationStats; + } + + private static Map<String, NodeMetricSnapshot> byHost(List<NodeTimeseries> allNodeTimeseries) { + Map<String, NodeMetricSnapshot> snapshots = new HashMap<>(); + for (var nodeTimeseries : allNodeTimeseries) + nodeTimeseries.last().ifPresent(last -> snapshots.put(nodeTimeseries.hostname(), last)); + return snapshots; + } + + private static double divide(double a, double b) { + if (a == 0 && b == 0) return 0; + return a / b; + } + + public static class ApplicationStats implements Comparable<ApplicationStats> { + + private final ApplicationId id; + private final Load load; + private final double cost; + private final double utilizedCost; + + public ApplicationStats(ApplicationId id, Load load, double cost, double utilizedCost) { + this.id = id; + this.load = load; + this.cost = cost; + this.utilizedCost = utilizedCost; + } + + public ApplicationId id() { return id; } + public Load load() { return load; } + + /** The standard cost of this application */ + public double cost() { return cost; } + + /** The amount of that cost which is currently utilized */ + public double utilizedCost() { return utilizedCost; } + + /** Cost - utilizedCost */ + public double unutilizedCost() { return cost - utilizedCost; } + + @Override + public int compareTo(NodeRepoStats.ApplicationStats other) { + return -Double.compare(this.unutilizedCost(), other.unutilizedCost()); + } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index f7ed4ebad3a..4cf531b55de 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.applications.Applications; +import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -30,12 +31,8 @@ import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import java.time.Clock; -import java.time.Duration; -import java.time.Instant; import java.util.List; import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -45,8 +42,6 @@ import java.util.stream.Collectors; */ public class NodeRepository extends AbstractComponent { - private static final Logger log = Logger.getLogger(NodeRepository.class.getName()); - private final CuratorDatabaseClient db; private final Clock clock; private final Zone zone; @@ -63,6 +58,7 @@ public class NodeRepository extends AbstractComponent { private final Applications applications; private final LoadBalancers loadBalancers; private final FlagSource flagSource; + private final MetricsDb metricsDb; private final int spareCount; /** @@ -75,7 +71,8 @@ public class NodeRepository extends AbstractComponent { ProvisionServiceProvider provisionServiceProvider, Curator curator, Zone zone, - FlagSource flagSource) { + FlagSource flagSource, + MetricsDb metricsDb) { this(flavors, provisionServiceProvider, curator, @@ -85,6 +82,7 @@ public class NodeRepository extends AbstractComponent { DockerImage.fromString(config.containerImage()) .withReplacedBy(DockerImage.fromString(config.containerImageReplacement())), flagSource, + metricsDb, config.useCuratorClientCache(), zone.environment().isProduction() && !zone.getCloud().dynamicProvisioning() ? 1 : 0, config.nodeCacheSize()); @@ -102,6 +100,7 @@ public class NodeRepository extends AbstractComponent { NameResolver nameResolver, DockerImage containerImage, FlagSource flagSource, + MetricsDb metricsDb, boolean useCuratorClientCache, int spareCount, long nodeCacheSize) { @@ -126,22 +125,9 @@ public class NodeRepository extends AbstractComponent { this.applications = new Applications(db); this.loadBalancers = new LoadBalancers(db); this.flagSource = flagSource; + this.metricsDb = metricsDb; this.spareCount = spareCount; - rewriteNodes(); - } - - /** Read and write all nodes to make sure they are stored in the latest version of the serialized format */ - private void rewriteNodes() { - Instant start = clock.instant(); - int nodesWritten = 0; - for (State state : State.values()) { - List<Node> nodes = db.readNodes(state); - // TODO(mpolden): This should take the lock before writing - db.writeTo(state, nodes, Agent.system, Optional.empty()); - nodesWritten += nodes.size(); - } - Instant end = clock.instant(); - log.log(Level.INFO, String.format("Rewrote %d nodes in %s", nodesWritten, Duration.between(start, end))); + nodes.rewrite(); } /** Returns the curator database client used by this */ @@ -183,6 +169,10 @@ public class NodeRepository extends AbstractComponent { public FlagSource flagSource() { return flagSource; } + public MetricsDb metricsDb() { return metricsDb; } + + public NodeRepoStats computeStats() { return NodeRepoStats.computeOver(this); } + /** Returns the time keeper of this system */ public Clock clock() { return clock; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index a06ad89e299..79b09348d21 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -96,18 +96,18 @@ public class AllocationOptimizer { // Memory and disk: Scales with group size // The fixed cost portion of cpu does not scale with changes to the node count - double queryCpuPerGroup = fixedCpuCostFraction * target.nodeCpu() + - (1 - fixedCpuCostFraction) * target.nodeCpu() * current.groupSize() / groupSize; + double queryCpuPerGroup = fixedCpuCostFraction * target.resources().vcpu() + + (1 - fixedCpuCostFraction) * target.resources().vcpu() * current.groupSize() / groupSize; double queryCpu = queryCpuPerGroup * current.groups() / groups; - double writeCpu = target.nodeCpu() * current.groupSize() / groupSize; + double writeCpu = target.resources().vcpu() * current.groupSize() / groupSize; cpu = clusterModel.queryCpuFraction() * queryCpu + (1 - clusterModel.queryCpuFraction()) * writeCpu; - memory = target.nodeMemory() * current.groupSize() / groupSize; - disk = target.nodeDisk() * current.groupSize() / groupSize; + memory = target.resources().memoryGb() * current.groupSize() / groupSize; + disk = target.resources().diskGb() * current.groupSize() / groupSize; } else { - cpu = target.nodeCpu() * current.nodes() / nodes; - memory = target.nodeMemory(); - disk = target.nodeDisk(); + cpu = target.resources().vcpu() * current.nodes() / nodes; + memory = target.resources().memoryGb(); + disk = target.resources().diskGb(); } // Combine the scaled resource values computed here diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 2a51a921a9f..1d0ba3da6c5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -26,12 +26,10 @@ public class Autoscaler { /** What resource difference is worth a reallocation? */ private static final double resourceDifferenceWorthReallocation = 0.1; - private final MetricsDb metricsDb; private final NodeRepository nodeRepository; private final AllocationOptimizer allocationOptimizer; - public Autoscaler(MetricsDb metricsDb, NodeRepository nodeRepository) { - this.metricsDb = metricsDb; + public Autoscaler(NodeRepository nodeRepository) { this.nodeRepository = nodeRepository; this.allocationOptimizer = new AllocationOptimizer(nodeRepository); } @@ -63,7 +61,7 @@ public class Autoscaler { cluster, clusterNodes.clusterSpec(), clusterNodes, - metricsDb, + nodeRepository.metricsDb(), nodeRepository.clock()); if ( ! clusterIsStable(clusterNodes, nodeRepository)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index acf227e3de2..e3622c8f076 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -96,15 +96,10 @@ public class ClusterModel { return queryFractionOfMax = clusterTimeseries().queryFractionOfMax(scalingDuration(), clock); } - public double averageLoad(Resource resource) { return nodeTimeseries().averageLoad(resource); } - - public double idealLoad(Resource resource) { - switch (resource) { - case cpu : return idealCpuLoad(); - case memory : return idealMemoryLoad; - case disk : return idealDiskLoad; - default : throw new IllegalStateException("No ideal load defined for " + resource); - } + public Load averageLoad() { return nodeTimeseries().averageLoad(); } + + public Load idealLoad() { + return new Load(idealCpuLoad(), idealMemoryLoad, idealDiskLoad); } /** Ideal cpu load must take the application traffic fraction into account */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index c097abd8208..a7396f29d92 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -42,21 +42,17 @@ public class ClusterNodesTimeseries { /** Returns the number of nodes measured in this */ public int nodesMeasured() { return timeseries.size(); } - /** Returns the average load of this resource in this */ - public double averageLoad(Resource resource) { - int measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); - if (measurementCount == 0) return 0; - double measurementSum = timeseries.stream().flatMap(m -> m.asList().stream()).mapToDouble(m -> value(resource, m)).sum(); - return measurementSum / measurementCount; - } - - private double value(Resource resource, NodeMetricSnapshot snapshot) { - switch (resource) { - case cpu: return snapshot.cpu(); - case memory: return snapshot.memory(); - case disk: return snapshot.disk(); - default: throw new IllegalArgumentException("Got an unknown resource " + resource); + /** Returns the average load in this */ + public Load averageLoad() { + Load total = Load.zero(); + int count = 0; + for (var nodeTimeseries : timeseries) { + for (var snapshot : nodeTimeseries.asList()) { + total = total.add(snapshot.load()); + count++; + } } + return total.divide(count); } private List<NodeTimeseries> filter(List<NodeTimeseries> timeseries, Predicate<NodeMetricSnapshot> filter) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java new file mode 100644 index 00000000000..1e400bd2627 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Load.java @@ -0,0 +1,79 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.NodeResources; + +/** + * The load of a node or system, measured as fractions of max (1.0) in three dimensions. + * + * @author bratseth + */ +public class Load { + + private final double cpu, memory, disk; + + public Load(double cpu, double memory, double disk) { + this.cpu = requireNormalized(cpu, "cpu"); + this.memory = requireNormalized(memory, "memory"); + this.disk = requireNormalized(disk, "disk"); + } + + public double cpu() { return cpu; } + public double memory() { return memory; } + public double disk() { return disk; } + + public Load add(Load other) { + return new Load(cpu + other.cpu(), memory + other.memory(), disk + other.disk()); + } + + public Load multiply(NodeResources resources) { + return new Load(cpu * resources.vcpu(), memory * resources.memoryGb(), disk * resources.diskGb()); + } + + public Load multiply(double factor) { + return new Load(cpu * factor, memory * factor, disk * factor); + } + + public Load divide(NodeResources resources) { + return new Load(divide(cpu, resources.vcpu()), divide(memory, resources.memoryGb()), divide(disk, resources.diskGb())); + } + + public Load divide(Load divisor) { + return new Load(divide(cpu, divisor.cpu()), divide(memory, divisor.memory()), divide(disk, divisor.disk())); + } + + public Load divide(double divisor) { + return new Load(divide(cpu, divisor), divide(memory, divisor), divide(disk, divisor)); + } + + public NodeResources scaled(NodeResources resources) { + return resources.withVcpu(cpu * resources.vcpu()) + .withMemoryGb(memory * resources.memoryGb()) + .withDiskGb(disk * resources.diskGb()); + } + + private double requireNormalized(double value, String name) { + if (Double.isNaN(value)) + throw new IllegalArgumentException(name + " must be a number but is NaN"); + if (value < 0) + throw new IllegalArgumentException(name + " must be zero or lager, but is " + value); + return value; + } + + @Override + public String toString() { + return "load: " + cpu + " cpu, " + memory + " memory, " + disk + " disk"; + } + + public static Load zero() { return new Load(0, 0, 0); } + + private static double divide(double a, double b) { + if (a == 0 && b == 0) return 0; + return a / b; + } + + public static Load byDividing(NodeResources a, NodeResources b) { + return new Load(divide(a.vcpu(), b.vcpu()), divide(a.memoryGb(), b.memoryGb()), divide(a.diskGb(), b.diskGb())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java index bf8d354665a..6b8ba3f7dc4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java @@ -27,7 +27,7 @@ import java.util.stream.Collectors; */ public class MemoryMetricsDb implements MetricsDb { - private final NodeRepository nodeRepository; + private final Clock clock; /** Metric time series by node (hostname). Each list of metric snapshots is sorted by increasing timestamp */ private final Map<String, NodeTimeseries> nodeTimeseries = new HashMap<>(); @@ -37,12 +37,12 @@ public class MemoryMetricsDb implements MetricsDb { /** Lock all access for now since we modify lists inside a map */ private final Object lock = new Object(); - public MemoryMetricsDb(NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; + public MemoryMetricsDb(Clock clock) { + this.clock = clock; } @Override - public Clock clock() { return nodeRepository.clock(); } + public Clock clock() { return clock; } @Override public void addNodeMetrics(Collection<Pair<String, NodeMetricSnapshot>> nodeMetrics) { @@ -70,11 +70,14 @@ public class MemoryMetricsDb implements MetricsDb { @Override public List<NodeTimeseries> getNodeTimeseries(Duration period, Set<String> hostnames) { - Instant startTime = nodeRepository.clock().instant().minus(period); + Instant startTime = clock().instant().minus(period); synchronized (lock) { - return hostnames.stream() - .map(hostname -> nodeTimeseries.getOrDefault(hostname, new NodeTimeseries(hostname, List.of())).justAfter(startTime)) - .collect(Collectors.toList()); + if (hostnames.isEmpty()) + return nodeTimeseries.values().stream().map(ns -> ns.justAfter(startTime)).collect(Collectors.toList()); + else + return hostnames.stream() + .map(hostname -> nodeTimeseries.getOrDefault(hostname, new NodeTimeseries(hostname, List.of())).justAfter(startTime)) + .collect(Collectors.toList()); } } @@ -91,7 +94,7 @@ public class MemoryMetricsDb implements MetricsDb { // 12 hours with 1k nodes and 3 resources and 1 measurement/sec is about 5Gb for (String hostname : nodeTimeseries.keySet()) { var timeseries = nodeTimeseries.get(hostname); - timeseries = timeseries.justAfter(nodeRepository.clock().instant().minus(Autoscaler.maxScalingWindow())); + timeseries = timeseries.justAfter(clock().instant().minus(Autoscaler.maxScalingWindow())); if (timeseries.isEmpty()) nodeTimeseries.remove(hostname); else @@ -106,9 +109,6 @@ public class MemoryMetricsDb implements MetricsDb { private void add(String hostname, NodeMetricSnapshot snapshot) { NodeTimeseries timeseries = nodeTimeseries.get(hostname); if (timeseries == null) { // new node - Optional<Node> node = nodeRepository.nodes().node(hostname); - if (node.isEmpty()) return; - if (node.get().allocation().isEmpty()) return; timeseries = new NodeTimeseries(hostname, new ArrayList<>()); nodeTimeseries.put(hostname, timeseries); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java index 568c5f88661..593b73e008e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsDb.java @@ -35,6 +35,7 @@ public interface MetricsDb { * the snapshots recorded after the given time (or an empty snapshot if none). * * @param period the duration into the past to return data for + * @param hostnames the host names to return timeseries for, or empty to return for all hostnames */ List<NodeTimeseries> getNodeTimeseries(Duration period, Set<String> hostnames); @@ -50,8 +51,8 @@ public interface MetricsDb { void close(); - static MemoryMetricsDb createTestInstance(NodeRepository nodeRepository) { - return new MemoryMetricsDb(nodeRepository); + static MemoryMetricsDb createTestInstance(Clock clock) { + return new MemoryMetricsDb(clock); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index b3cf6c1e962..210388db7b8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -67,9 +67,9 @@ public class MetricsResponse { consumeServiceMetrics(nodeObject.field("services"), nodeValues); nodeMetrics.add(new Pair<>(hostname, new NodeMetricSnapshot(at, - Metric.cpu.from(nodeValues), - Metric.memory.from(nodeValues), - Metric.disk.from(nodeValues), + new Load(Metric.cpu.from(nodeValues), + Metric.memory.from(nodeValues), + Metric.disk.from(nodeValues)), (long)Metric.generation.from(nodeValues), Metric.inService.from(nodeValues) > 0, clusterIsStable(node.get(), applicationNodes, nodeRepository), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java index b2d8ddfd414..5c45e4a3737 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcher.java @@ -1,7 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; -import ai.vespa.util.http.VespaAsyncHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java index be9f7bd4819..6329d350642 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricSnapshot.java @@ -12,21 +12,17 @@ public class NodeMetricSnapshot implements Comparable<NodeMetricSnapshot> { private final Instant at; - private final double cpu; - private final double memory; - private final double disk; + private final Load load; private final long generation; private final boolean inService; private final boolean stable; private final double queryRate; - public NodeMetricSnapshot(Instant at, double cpu, double memory, double disk, + public NodeMetricSnapshot(Instant at, Load load, long generation, boolean inService, boolean stable, double queryRate) { this.at = at; - this.cpu = cpu; - this.memory = memory; - this.disk = disk; + this.load = load; this.generation = generation; this.inService = inService; this.stable = stable; @@ -34,9 +30,7 @@ public class NodeMetricSnapshot implements Comparable<NodeMetricSnapshot> { } public Instant at() { return at; } - public double cpu() { return cpu; } - public double memory() { return memory; } - public double disk() { return disk; } + public Load load() { return load; } /** Queries per second */ public double queryRate() { return queryRate; } @@ -53,10 +47,8 @@ public class NodeMetricSnapshot implements Comparable<NodeMetricSnapshot> { } @Override - public String toString() { return "metrics at " + at + ":" + - " cpu: " + cpu + - " memory: " + memory + - " disk: " + disk + + public String toString() { return "metrics at " + at + ": " + + load + " generation: " + generation + " inService: " + inService + " stable: " + stable + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index cedc2edfe63..f8100677b10 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -5,6 +5,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -31,6 +32,12 @@ public class NodeTimeseries { public NodeMetricSnapshot get(int index) { return snapshots.get(index); } + /** Returns the last (newest) snapshot in this, or empty if there are none. */ + public Optional<NodeMetricSnapshot> last() { + if (snapshots.isEmpty()) return Optional.empty(); + return Optional.of(snapshots.get(snapshots.size() - 1)); + } + public List<NodeMetricSnapshot> asList() { return snapshots; } public String hostname() { return hostname; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index fe97df0e876..c933e16041a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -113,9 +113,9 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { TableWriter.Row row = writer.newRow(atMillis * 1000); // in microseconds row.putStr(0, snapshot.getFirst()); // (1 is timestamp) - row.putFloat(2, (float)snapshot.getSecond().cpu()); - row.putFloat(3, (float)snapshot.getSecond().memory()); - row.putFloat(4, (float)snapshot.getSecond().disk()); + row.putFloat(2, (float)snapshot.getSecond().load().cpu()); + row.putFloat(3, (float)snapshot.getSecond().load().memory()); + row.putFloat(4, (float)snapshot.getSecond().load().disk()); row.putLong(5, snapshot.getSecond().generation()); row.putBool(6, snapshot.getSecond().inService()); row.putBool(7, snapshot.getSecond().stable()); @@ -355,12 +355,12 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { Record record = cursor.getRecord(); while (cursor.hasNext()) { String hostname = record.getStr(0).toString(); - if (hostnames.contains(hostname)) { + if (hostnames.isEmpty() || hostnames.contains(hostname)) { snapshots.put(hostname, new NodeMetricSnapshot(Instant.ofEpochMilli(record.getTimestamp(1) / 1000), - record.getFloat(2), - record.getFloat(3), - record.getFloat(4), + new Load(record.getFloat(2), + record.getFloat(3), + record.getFloat(4)), record.getLong(5), record.getBool(6), record.getBool(7), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java deleted file mode 100644 index c639ad1f779..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; - -import com.yahoo.config.provision.NodeResources; - -/** - * A resource subject to autoscaling - * - * @author bratseth - */ -public enum Resource { - - /** Cpu utilization ratio */ - cpu { - double valueFrom(NodeResources resources) { return resources.vcpu(); } - }, - - /** Memory utilization ratio */ - memory { - double valueFrom(NodeResources resources) { return resources.memoryGb(); } - }, - - /** Disk utilization ratio */ - disk { - double valueFrom(NodeResources resources) { return resources.diskGb(); } - }; - - abstract double valueFrom(NodeResources resources); - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java index b1a1e86b08d..3b8b7f08684 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.applications.Application; import java.time.Clock; @@ -18,56 +19,34 @@ public class ResourceTarget { private final boolean adjustForRedundancy; /** The target real resources per node, assuming the node assignment where this was decided */ - private final double cpu, memory, disk; + private final NodeResources resources; - private ResourceTarget(double cpu, double memory, double disk, boolean adjustForRedundancy) { - this.cpu = cpu; - this.memory = memory; - this.disk = disk; + private ResourceTarget(NodeResources resources, boolean adjustForRedundancy) { + this.resources = resources; this.adjustForRedundancy = adjustForRedundancy; } /** Are the target resources given by this including redundancy or not */ public boolean adjustForRedundancy() { return adjustForRedundancy; } - /** Returns the target cpu per node, in terms of the current allocation */ - public double nodeCpu() { return cpu; } - - /** Returns the target memory per node, in terms of the current allocation */ - public double nodeMemory() { return memory; } - - /** Returns the target disk per node, in terms of the current allocation */ - public double nodeDisk() { return disk; } + /** Returns the target resources per node in terms of the current allocation */ + public NodeResources resources() { return resources; } @Override public String toString() { - return "target " + - (adjustForRedundancy ? "(with redundancy adjustment) " : "") + - "[vcpu " + cpu + ", memoryGb " + memory + ", diskGb " + disk + "]"; - } - - private static double nodeUsage(Resource resource, double load, AllocatableClusterResources current) { - return load * resource.valueFrom(current.realResources().nodeResources()); + return "target " + resources + (adjustForRedundancy ? "(with redundancy adjustment) " : ""); } /** Create a target of achieving ideal load given a current load */ public static ResourceTarget idealLoad(ClusterModel clusterModel, AllocatableClusterResources current) { - return new ResourceTarget(nodeUsage(Resource.cpu, clusterModel.averageLoad(Resource.cpu), current) - / clusterModel.idealLoad(Resource.cpu), - nodeUsage(Resource.memory, clusterModel.averageLoad(Resource.memory), current) - / clusterModel.idealLoad(Resource.memory), - nodeUsage(Resource.disk, clusterModel.averageLoad(Resource.disk), current) - / clusterModel.idealLoad(Resource.disk), - true); + var loadAdjustment = clusterModel.averageLoad().divide(clusterModel.idealLoad()); + return new ResourceTarget(loadAdjustment.scaled(current.realResources().nodeResources()), true); } /** Crete a target of preserving a current allocation */ public static ResourceTarget preserve(AllocatableClusterResources current) { - return new ResourceTarget(current.realResources().nodeResources().vcpu(), - current.realResources().nodeResources().memoryGb(), - current.realResources().nodeResources().diskGb(), - false); + return new ResourceTarget(current.realResources().nodeResources(), false); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 40d495a47ee..17d33ef501c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -33,18 +33,15 @@ import java.util.Set; public class AutoscalingMaintainer extends NodeRepositoryMaintainer { private final Autoscaler autoscaler; - private final MetricsDb metricsDb; private final Deployer deployer; private final Metric metric; public AutoscalingMaintainer(NodeRepository nodeRepository, - MetricsDb metricsDb, Deployer deployer, Metric metric, Duration interval) { super(nodeRepository, interval, metric); - this.autoscaler = new Autoscaler(metricsDb, nodeRepository); - this.metricsDb = metricsDb; + this.autoscaler = new Autoscaler(nodeRepository); this.deployer = deployer; this.metric = metric; } @@ -115,8 +112,8 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { .anyMatch(node -> node.history().hasEventAt(History.Event.Type.retired, event.at()))) return cluster; // - 2. all nodes have switched to the right config generation - for (NodeTimeseries nodeTimeseries : metricsDb.getNodeTimeseries(Duration.between(event.at(), clock().instant()), - clusterNodes)) { + for (var nodeTimeseries : nodeRepository().metricsDb().getNodeTimeseries(Duration.between(event.at(), clock().instant()), + clusterNodes)) { Optional<NodeMetricSnapshot> firstOnNewGeneration = nodeTimeseries.asList().stream() .filter(snapshot -> snapshot.generation() >= event.generation()).findFirst(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index f4509c0713e..f95094f891c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -24,16 +24,13 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { private static final int maxWarningsPerInvocation = 2; private final MetricsFetcher metricsFetcher; - private final MetricsDb metricsDb; public NodeMetricsDbMaintainer(NodeRepository nodeRepository, MetricsFetcher metricsFetcher, - MetricsDb metricsDb, Duration interval, Metric metric) { super(nodeRepository, interval, metric); this.metricsFetcher = metricsFetcher; - this.metricsDb = metricsDb; } @Override @@ -54,7 +51,7 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { if (++done < applications.size()) Thread.sleep(pauseMs); } - metricsDb.gc(); + nodeRepository().metricsDb().gc(); // Suppress failures for manual zones for now to avoid noise return nodeRepository().zone().environment().isManuallyDeployed() || warnings.get() == 0; @@ -75,8 +72,8 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { warnings.add(1); } else if (response != null) { - metricsDb.addNodeMetrics(response.nodeMetrics()); - metricsDb.addClusterMetrics(application, response.clusterMetrics()); + nodeRepository().metricsDb().addNodeMetrics(response.nodeMetrics()); + nodeRepository().metricsDb().addClusterMetrics(application, response.clusterMetrics()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 7f41f89f664..46230ed38a4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -39,7 +39,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, - MetricsFetcher metricsFetcher, MetricsDb metricsDb) { + MetricsFetcher metricsFetcher) { DefaultTimes defaults = new DefaultTimes(zone, deployer); PeriodicApplicationMaintainer periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, @@ -64,9 +64,9 @@ public class NodeRepositoryMaintenance extends AbstractComponent { maintainers.add(new SpareCapacityMaintainer(deployer, nodeRepository, metric, defaults.spareCapacityMaintenanceInterval)); maintainers.add(new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval, metric)); maintainers.add(new Rebalancer(deployer, nodeRepository, metric, defaults.rebalancerInterval)); - maintainers.add(new NodeMetricsDbMaintainer(nodeRepository, metricsFetcher, metricsDb, defaults.nodeMetricsCollectionInterval, metric)); - maintainers.add(new AutoscalingMaintainer(nodeRepository, metricsDb, deployer, metric, defaults.autoscalingInterval)); - maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, metricsDb, defaults.scalingSuggestionsInterval, metric)); + maintainers.add(new NodeMetricsDbMaintainer(nodeRepository, metricsFetcher, defaults.nodeMetricsCollectionInterval, metric)); + maintainers.add(new AutoscalingMaintainer(nodeRepository, deployer, metric, defaults.autoscalingInterval)); + maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, defaults.scalingSuggestionsInterval, metric)); maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer)); provisionServiceProvider.getLoadBalancerService(nodeRepository) @@ -150,12 +150,14 @@ public class NodeRepositoryMaintenance extends AbstractComponent { inactiveConfigServerExpiry = Duration.ofMinutes(5); inactiveControllerExpiry = Duration.ofMinutes(5); - if (zone.environment() == Environment.prod && ! zone.system().isCd()) { + if (zone.environment().isProduction() && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy retiredInterval = Duration.ofMinutes(30); dirtyExpiry = Duration.ofHours(2); // enough time to clean the node } else { - inactiveExpiry = Duration.ofSeconds(2); // support interactive wipe start over + // long enough that nodes aren't reused immediately and delete can happen on all config servers + // with time enough to clean up even with ZK connection issues on config servers + inactiveExpiry = Duration.ofMinutes(1); retiredInterval = Duration.ofMinutes(1); dirtyExpiry = Duration.ofMinutes(30); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index e2b89879141..c217580872b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -29,11 +29,10 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { private final Autoscaler autoscaler; public ScalingSuggestionsMaintainer(NodeRepository nodeRepository, - MetricsDb metricsDb, Duration interval, Metric metric) { super(nodeRepository, interval, metric); - this.autoscaler = new Autoscaler(metricsDb, nodeRepository); + this.autoscaler = new Autoscaler(nodeRepository); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index cb9fb8182d4..f12699e0d81 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; @@ -33,6 +34,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.BiFunction; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -51,6 +54,8 @@ import java.util.stream.Stream; // Nodes might have an application assigned in dirty. public class Nodes { + private static final Logger log = Logger.getLogger(Nodes.class.getName()); + private final Zone zone; private final Clock clock; private final CuratorDatabaseClient db; @@ -61,6 +66,20 @@ public class Nodes { this.db = db; } + /** Read and write all nodes to make sure they are stored in the latest version of the serialized format */ + public void rewrite() { + Instant start = clock.instant(); + int nodesWritten = 0; + for (Node.State state : Node.State.values()) { + List<Node> nodes = db.readNodes(state); + // TODO(mpolden): This should take the lock before writing + db.writeTo(state, nodes, Agent.system, Optional.empty()); + nodesWritten += nodes.size(); + } + Instant end = clock.instant(); + log.log(Level.INFO, String.format("Rewrote %d nodes in %s", nodesWritten, Duration.between(start, end))); + } + // ---------------- Query API ---------------------------------------------------------------- /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 75fa3aec1e2..a54acbe52ae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -52,7 +52,6 @@ public class NodeRepositoryProvisioner implements Provisioner { private static final Logger log = Logger.getLogger(NodeRepositoryProvisioner.class.getName()); private final NodeRepository nodeRepository; - private final MetricsDb metricsDb; private final AllocationOptimizer allocationOptimizer; private final CapacityPolicies capacityPolicies; private final Zone zone; @@ -63,11 +62,9 @@ public class NodeRepositoryProvisioner implements Provisioner { @Inject public NodeRepositoryProvisioner(NodeRepository nodeRepository, - MetricsDb metricsDb, Zone zone, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) { this.nodeRepository = nodeRepository; - this.metricsDb = metricsDb; this.allocationOptimizer = new AllocationOptimizer(nodeRepository); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; @@ -167,7 +164,7 @@ public class NodeRepositoryProvisioner implements Provisioner { firstDeployment // start at min, preserve current resources otherwise ? new AllocatableClusterResources(requested.minResources(), clusterSpec, nodeRepository) : new AllocatableClusterResources(nodes.asList(), nodeRepository, clusterSpec.isExclusive()); - var clusterModel = new ClusterModel(application, cluster, clusterSpec, nodes, metricsDb, nodeRepository.clock()); + var clusterModel = new ClusterModel(application, cluster, clusterSpec, nodes, nodeRepository.metricsDb(), nodeRepository.clock()); return within(Limits.of(requested), currentResources, firstDeployment, clusterModel); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 176bf195f1f..692d757f41d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -1,27 +1,18 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.autoscale.ClusterModel; -import com.yahoo.vespa.hosted.provision.autoscale.ClusterNodesTimeseries; -import com.yahoo.vespa.hosted.provision.autoscale.ClusterTimeseries; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; -import com.yahoo.vespa.hosted.provision.autoscale.Resource; -import com.yahoo.vespa.hosted.provision.autoscale.ResourceTarget; import java.net.URI; -import java.time.Clock; -import java.time.Duration; -import java.util.Collection; import java.util.List; /** @@ -94,12 +85,12 @@ public class ApplicationSerializer { } private static void clusterUtilizationToSlime(ClusterModel clusterModel, Cursor utilizationObject) { - utilizationObject.setDouble("cpu", clusterModel.averageLoad(Resource.cpu)); - utilizationObject.setDouble("idealCpu", clusterModel.idealLoad(Resource.cpu)); - utilizationObject.setDouble("memory", clusterModel.averageLoad(Resource.memory)); - utilizationObject.setDouble("idealMemory", clusterModel.idealLoad(Resource.memory)); - utilizationObject.setDouble("disk", clusterModel.averageLoad(Resource.disk)); - utilizationObject.setDouble("idealDisk", clusterModel.idealLoad(Resource.disk)); + utilizationObject.setDouble("cpu", clusterModel.averageLoad().cpu()); + utilizationObject.setDouble("idealCpu", clusterModel.idealLoad().cpu()); + utilizationObject.setDouble("memory", clusterModel.averageLoad().memory()); + utilizationObject.setDouble("idealMemory", clusterModel.idealLoad().memory()); + utilizationObject.setDouble("disk", clusterModel.averageLoad().disk()); + utilizationObject.setDouble("idealDisk", clusterModel.idealLoad().disk()); } private static void scalingEventsToSlime(List<ScalingEvent> scalingEvents, Cursor scalingEventsArray) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 32952eeb860..6d27acf77d1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -68,6 +68,7 @@ public class MockNodeRepository extends NodeRepository { new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), new InMemoryFlagSource(), + new MemoryMetricsDb(Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))), true, 0, 1000); this.flavors = flavors; @@ -78,7 +79,6 @@ public class MockNodeRepository extends NodeRepository { private void populate() { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, - new MemoryMetricsDb(this), Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java new file mode 100644 index 00000000000..44376fc103c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java @@ -0,0 +1,132 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision; + +import com.yahoo.collections.Pair; +import com.yahoo.component.Version; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.autoscale.Load; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class NodeRepoStatsTest { + + private static final double delta = 0.0001; + + @Test + public void testEmpty() { + var tester = new NodeRepositoryTester(); + assertLoad(Load.zero(), tester.nodeRepository().computeStats().load()); + assertLoad(Load.zero(), tester.nodeRepository().computeStats().activeLoad()); + assertTrue(tester.nodeRepository().computeStats().applicationStats().isEmpty()); + } + + @Test + public void testHostButNoNodes() { + var tester = new NodeRepositoryTester(); + tester.addHost("host1", "default"); + tester.addHost("host2", "default"); + tester.addHost("host3", "small"); + assertLoad(Load.zero(), tester.nodeRepository().computeStats().load()); + assertLoad(Load.zero(), tester.nodeRepository().computeStats().activeLoad()); + assertTrue(tester.nodeRepository().computeStats().applicationStats().isEmpty()); + } + + @Test + public void testStats() { + var hostResources = new NodeResources(10, 100, 1000, 10, + NodeResources.DiskSpeed.fast, NodeResources.StorageType.local); + + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(10, hostResources).activateTenantHosts(); + + var app1 = ProvisioningTester.applicationId("app1"); + var app2 = ProvisioningTester.applicationId("app2"); + var app3 = ProvisioningTester.applicationId("app3"); + var cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("cluster1")).vespaVersion(Version.fromString("7")).build(); + var cluster2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("cluster2")).vespaVersion(Version.fromString("7")).build(); + var small = new NodeResources(1, 10, 100, 1, NodeResources.DiskSpeed.any); + var large = new NodeResources(2, 20, 200, 1); + + // Deploy apps + var hostsApp1 = tester.prepare(app1, cluster1, Capacity.from(new ClusterResources(6, 1, small))); + hostsApp1.addAll(tester.prepare(app1, cluster2, Capacity.from(new ClusterResources(4, 1, large)))); + tester.activate(app1, hostsApp1); + tester.activate(app2, cluster1, Capacity.from(new ClusterResources(8, 1, small))); + tester.activate(app3, cluster1, Capacity.from(new ClusterResources(5, 1, large))); + + // Add metrics + double loadApp1Cluster1 = 0.2; + double loadApp1Cluster2 = 0.3; + double loadApp2 = 0.4; + double loadApp3 = 0.5; + var now = tester.clock().instant(); + for (Node node : tester.nodeRepository().nodes().list(Node.State.active)) { + double loadFactor; + var allocation = node.allocation().get(); + if (allocation.owner().equals(app1)) { + if (allocation.membership().cluster().id().equals(cluster1.id())) + loadFactor = loadApp1Cluster1; + else + loadFactor = loadApp1Cluster2; + } + else if (allocation.owner().equals(app2)) { + loadFactor = loadApp2; + } + else { + loadFactor = loadApp3; + } + var snapshot = new NodeMetricSnapshot(now, new Load(1.0, 0.9, 0.8).multiply(loadFactor), 1, true, true, 1.0 ); + tester.nodeRepository().metricsDb().addNodeMetrics(List.of(new Pair<>(node.hostname(), snapshot))); + } + + var stats = tester.nodeRepository().computeStats(); + + assertLoad(new Load(0.6180,0.5562,0.4944), stats.load()); + assertLoad(new Load(0.4682,0.4214,0.3745), stats.activeLoad()); + + var app1Stats = stats.applicationStats().get(0); + var app2Stats = stats.applicationStats().get(2); + var app3Stats = stats.applicationStats().get(1); + + assertEquals(app1, app1Stats.id()); + assertEquals(2.940, app1Stats.cost(), delta); + assertEquals(0.702, app1Stats.utilizedCost(), delta); + assertEquals(2.238, app1Stats.unutilizedCost(), delta); + assertLoad(new Load(0.2571, 0.2314, 0.2057), app1Stats.load()); + + assertEquals(app2, app2Stats.id()); + assertEquals(1.680, app2Stats.cost(), delta); + assertEquals(0.624, app2Stats.utilizedCost(), delta); + assertEquals(1.056, app2Stats.unutilizedCost(), delta); + assertLoad(new Load(.40, 0.36, 0.32), app2Stats.load()); + + assertEquals(app3, app3Stats.id()); + assertEquals(2.100, app3Stats.cost(), delta); + assertEquals(0.975, app3Stats.utilizedCost(), delta); + assertEquals(1.125, app3Stats.unutilizedCost(), delta); + assertLoad(new Load(0.5, 0.45, 0.40), app3Stats.load()); + } + + private static void assertLoad(Load expected, Load actual) { + assertEquals("cpu", expected.cpu(), actual.cpu(), delta); + assertEquals("memory", expected.memory(), actual.memory(), delta); + assertEquals("disk", expected.disk(), actual.disk(), delta); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 1c03dcadbd4..c986a5df1d3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -4,12 +4,14 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; @@ -43,6 +45,7 @@ public class NodeRepositoryTester { new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), new InMemoryFlagSource(), + new MemoryMetricsDb(clock), true, 0, 1000); } @@ -54,10 +57,18 @@ public class NodeRepositoryTester { return nodeRepository.nodes().list(inState).nodeType(type).asList(); } + public Node addHost(String id, String flavor) { + return addNode(id, id, null, nodeFlavors.getFlavorOrThrow(flavor), NodeType.host); + } + public Node addHost(String id, String hostname, String flavor, NodeType type) { return addNode(id, hostname, null, nodeFlavors.getFlavorOrThrow(flavor), type); } + public Node addNode(String id, String parentHostname, NodeResources resources) { + return addNode(id, id, parentHostname, new Flavor(resources), NodeType.tenant); + } + public Node addNode(String id, String hostname, String parentHostname, String flavor, NodeType type) { return addNode(id, hostname, parentHostname, nodeFlavors.getFlavorOrThrow(flavor), type); } @@ -69,13 +80,16 @@ public class NodeRepositoryTester { return nodeRepository.nodes().addNodes(List.of(node), Agent.system).get(0); } + public void setNodeState(String hostname, Node.State state) { + setNodeState(nodeRepository.nodes().node(hostname).orElseThrow(RuntimeException::new), state); + } + /** * Moves a node directly to the given state without doing any validation, useful * to create wanted test scenario without having to move every node through series * of valid state transitions */ - public void setNodeState(String hostname, Node.State state) { - Node node = nodeRepository.nodes().node(hostname).orElseThrow(RuntimeException::new); + public void setNodeState(Node node, Node.State state) { nodeRepository.database().writeTo(state, node, Agent.system, Optional.empty()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 8c6c116a225..2eab0453d58 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -34,7 +34,7 @@ public class AutoscalingIntegrationTest { MetricsV2MetricsFetcher fetcher = new MetricsV2MetricsFetcher(tester.nodeRepository(), new OrchestratorMock(), new MockHttpClient(tester.clock())); - Autoscaler autoscaler = new Autoscaler(tester.nodeMetricsDb(), tester.nodeRepository()); + Autoscaler autoscaler = new Autoscaler(tester.nodeRepository()); ApplicationId application1 = tester.applicationId("test1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "test"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 89da20c5550..5fe6023e5af 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -46,7 +46,6 @@ class AutoscalingTester { private final ProvisioningTester provisioningTester; private final Autoscaler autoscaler; - private final MemoryMetricsDb db; private final MockHostResourcesCalculator hostResourcesCalculator; /** Creates an autoscaling tester with a single host type ready */ @@ -73,8 +72,7 @@ class AutoscalingTester { .build(); hostResourcesCalculator = new MockHostResourcesCalculator(zone); - db = MetricsDb.createTestInstance(provisioningTester.nodeRepository()); - autoscaler = new Autoscaler(db, nodeRepository()); + autoscaler = new Autoscaler(nodeRepository()); } public ProvisioningTester provisioning() { return provisioningTester; } @@ -140,17 +138,16 @@ class AutoscalingTester { for (int i = 0; i < count; i++) { clock().advance(Duration.ofMinutes(5)); for (Node node : nodes) { - float cpu = value * oneExtraNodeFactor; - float memory = (float) ClusterModel.idealMemoryLoad * otherResourcesLoad * oneExtraNodeFactor; - float disk = (float) ClusterModel.idealDiskLoad * otherResourcesLoad * oneExtraNodeFactor; - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - memory, - disk, - 0, - true, - true, - 0.0)))); + Load load = new Load(value, + ClusterModel.idealMemoryLoad * otherResourcesLoad, + ClusterModel.idealDiskLoad * otherResourcesLoad).multiply(oneExtraNodeFactor); + nodeMetricsDb().addNodeMetrics(List.of(new Pair<>(node.hostname(), + new NodeMetricSnapshot(clock().instant(), + load, + 0, + true, + true, + 0.0)))); } } } @@ -175,14 +172,16 @@ class AutoscalingTester { float cpu = (float) 0.2 * otherResourcesLoad * oneExtraNodeFactor; float memory = value * oneExtraNodeFactor; float disk = (float) ClusterModel.idealDiskLoad * otherResourcesLoad * oneExtraNodeFactor; - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - memory, - disk, - 0, - true, - true, - 0.0)))); + Load load = new Load(0.2 * otherResourcesLoad, + value, + ClusterModel.idealDiskLoad * otherResourcesLoad).multiply(oneExtraNodeFactor); + nodeMetricsDb().addNodeMetrics(List.of(new Pair<>(node.hostname(), + new NodeMetricSnapshot(clock().instant(), + load, + 0, + true, + true, + 0.0)))); } } } @@ -197,14 +196,13 @@ class AutoscalingTester { for (int i = 0; i < count; i++) { clock().advance(Duration.ofMinutes(1)); for (Node node : nodes) { - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - memory, - disk, - generation, - inService, - stable, - 0.0)))); + nodeMetricsDb().addNodeMetrics(List.of(new Pair<>(node.hostname(), + new NodeMetricSnapshot(clock().instant(), + new Load(cpu, memory, disk), + generation, + inService, + stable, + 0.0)))); } } } @@ -242,7 +240,10 @@ class AutoscalingTester { IntFunction<Double> queryRate, IntFunction<Double> writeRate) { for (int i = 0; i < measurements; i++) { - db.addClusterMetrics(application, Map.of(cluster, new ClusterMetricSnapshot(clock().instant(), queryRate.apply(i), writeRate.apply(i)))); + nodeMetricsDb().addClusterMetrics(application, + Map.of(cluster, new ClusterMetricSnapshot(clock().instant(), + queryRate.apply(i), + writeRate.apply(i)))); clock().advance(Duration.ofMinutes(5)); } } @@ -253,13 +254,16 @@ class AutoscalingTester { int measurements, IntFunction<Double> queryRate) { for (int i = 0; i < measurements; i++) { - db.addClusterMetrics(application, Map.of(cluster, new ClusterMetricSnapshot(clock().instant(), queryRate.apply(i), 0.0))); + nodeMetricsDb().addClusterMetrics(application, + Map.of(cluster, new ClusterMetricSnapshot(clock().instant(), + queryRate.apply(i), + 0.0))); clock().advance(Duration.ofMinutes(5)); } } public void clearQueryRateMeasurements(ApplicationId application, ClusterSpec.Id cluster) { - db.clearClusterMetrics(application, cluster); + ((MemoryMetricsDb)nodeMetricsDb()).clearClusterMetrics(application, cluster); } public Autoscaler.Advice autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, @@ -307,7 +311,7 @@ class AutoscalingTester { return provisioningTester.nodeRepository(); } - public MetricsDb nodeMetricsDb() { return db; } + public MetricsDb nodeMetricsDb() { return nodeRepository().metricsDb(); } private static class MockHostResourcesCalculator implements HostResourcesCalculator { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java index 70550b0a7c3..550ecceee23 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java @@ -37,13 +37,13 @@ public class ClusterModelTest { var model1 = new ClusterModel(application.with(new Status(0.0, 1.0)), cluster, clock, Duration.ofMinutes(10), timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock)); - assertEquals(0.131, model1.idealLoad(Resource.cpu), delta); + assertEquals(0.131, model1.idealLoad().cpu(), delta); // Almost no current traffic share: Ideal load is low but capped var model2 = new ClusterModel(application.with(new Status(0.0001, 1.0)), cluster, clock, Duration.ofMinutes(10), timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock)); - assertEquals(0.131, model2.idealLoad(Resource.cpu), delta); + assertEquals(0.131, model2.idealLoad().cpu(), delta); } @Test @@ -58,13 +58,13 @@ public class ClusterModelTest { var model1 = new ClusterModel(application, cluster, clock, Duration.ofMinutes(10), timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0, t -> 0.0, clock)); - assertEquals(0.275, model1.idealLoad(Resource.cpu), delta); + assertEquals(0.275, model1.idealLoad().cpu(), delta); - // Almost current traffic: Ideal load is low but capped + // Almost no current traffic: Ideal load is low but capped var model2 = new ClusterModel(application.with(new Status(0.0001, 1.0)), cluster, clock, Duration.ofMinutes(10), timeseries(cluster,100, t -> t == 0 ? 10000.0 : 0.0001, t -> 0.0, clock)); - assertEquals(0.275, model1.idealLoad(Resource.cpu), delta); + assertEquals(0.040, model2.idealLoad().cpu(), delta); } private Cluster cluster(NodeResources resources) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java index ef85e228cc7..5f1a36e7b56 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java @@ -51,14 +51,14 @@ public class MetricsV2MetricsFetcherTest { assertEquals(2, values.size()); assertEquals("host-1.yahoo.com", values.get(0).getFirst()); - assertEquals(0.162, values.get(0).getSecond().cpu(), delta); - assertEquals(0.231, values.get(0).getSecond().memory(), delta); - assertEquals(0.820, values.get(0).getSecond().disk(), delta); + assertEquals(0.162, values.get(0).getSecond().load().cpu(), delta); + assertEquals(0.231, values.get(0).getSecond().load().memory(), delta); + assertEquals(0.820, values.get(0).getSecond().load().disk(), delta); assertEquals("host-2.yahoo.com", values.get(1).getFirst()); - assertEquals(0.2, values.get(1).getSecond().cpu(), delta); - assertEquals(0.0, values.get(1).getSecond().memory(), delta); - assertEquals(0.4, values.get(1).getSecond().disk(), delta); + assertEquals(0.2, values.get(1).getSecond().load().cpu(), delta); + assertEquals(0.0, values.get(1).getSecond().load().memory(), delta); + assertEquals(0.4, values.get(1).getSecond().load().disk(), delta); assertEquals(45.0, values.get(1).getSecond().queryRate(), delta); } @@ -69,9 +69,9 @@ public class MetricsV2MetricsFetcherTest { httpClient.requestsReceived.get(1)); assertEquals(1, values.size()); assertEquals("host-3.yahoo.com", values.get(0).getFirst()); - assertEquals(0.10, values.get(0).getSecond().cpu(), delta); - assertEquals(0.15, values.get(0).getSecond().memory(), delta); - assertEquals(0.20, values.get(0).getSecond().disk(), delta); + assertEquals(0.10, values.get(0).getSecond().load().cpu(), delta); + assertEquals(0.15, values.get(0).getSecond().load().memory(), delta); + assertEquals(0.20, values.get(0).getSecond().load().disk(), delta); assertEquals(3, values.get(0).getSecond().generation(), delta); assertTrue(values.get(0).getSecond().stable()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 76e56004871..ae14b94e619 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -37,27 +37,24 @@ public class NodeMetricsDbTest { String node0 = hosts.iterator().next().hostname(); ManualClock clock = tester.clock(); - MetricsDb db = MetricsDb.createTestInstance(tester.nodeRepository()); Collection<Pair<String, NodeMetricSnapshot>> values = new ArrayList<>(); for (int i = 0; i < 40; i++) { values.add(new Pair<>(node0, new NodeMetricSnapshot(clock.instant(), - 0.9f, - 0.6f, - 0.6f, + new Load(0.9, 0.6, 0.6), 0, true, false, 0.0))); clock.advance(Duration.ofMinutes(120)); } - db.addNodeMetrics(values); + tester.nodeRepository().metricsDb().addNodeMetrics(values); // Avoid off-by-one bug when the below windows starts exactly on one of the above getEpochSecond() timestamps. clock.advance(Duration.ofMinutes(1)); - assertEquals(35, measurementCount(db.getNodeTimeseries(Duration.ofHours(72), Set.of(node0)))); - db.gc(); - assertEquals(23, measurementCount(db.getNodeTimeseries(Duration.ofHours(72), Set.of(node0)))); + assertEquals(35, measurementCount(tester.nodeRepository().metricsDb().getNodeTimeseries(Duration.ofHours(72), Set.of(node0)))); + tester.nodeRepository().metricsDb().gc(); + assertEquals(23, measurementCount(tester.nodeRepository().metricsDb().getNodeTimeseries(Duration.ofHours(72), Set.of(node0)))); } private int measurementCount(List<NodeTimeseries> measurements) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index f465a57d76a..34243f4548f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -54,9 +54,9 @@ public class QuestMetricsDbTest { assertEquals(1000, nodeTimeSeries1.get(0).size()); NodeMetricSnapshot snapshot = nodeTimeSeries1.get(0).asList().get(0); assertEquals(startTime.plus(Duration.ofSeconds(1)), snapshot.at()); - assertEquals(0.1, snapshot.cpu(), delta); - assertEquals(0.2, snapshot.memory(), delta); - assertEquals(0.4, snapshot.disk(), delta); + assertEquals(0.1, snapshot.load().cpu(), delta); + assertEquals(0.2, snapshot.load().memory(), delta); + assertEquals(0.4, snapshot.load().disk(), delta); assertEquals(1, snapshot.generation(), delta); assertEquals(30, snapshot.queryRate(), delta); @@ -234,10 +234,8 @@ public class QuestMetricsDbTest { for (int i = 1; i <= countPerHost; i++) { for (String host : hosts) timeseries.add(new Pair<>(host, new NodeMetricSnapshot(clock.instant(), - i * 0.1, - i * 0.2, - i * 0.4, - i % 100, + new Load(i * 0.1, i * 0.2, i * 0.4), + i % 100, true, true, 30.0))); @@ -260,11 +258,8 @@ public class QuestMetricsDbTest { Collection<Pair<String, NodeMetricSnapshot>> timeseries = new ArrayList<>(); for (int i = 1; i <= countPerHost; i++) { for (String host : hosts) - timeseries.add(new Pair<>(host, new NodeMetricSnapshot(at, - i * 0.1, - i * 0.2, - i * 0.4, - i % 100, + timeseries.add(new Pair<>(host, new NodeMetricSnapshot(at, new Load(i * 0.1, i * 0.2, i * 0.4), + i % 100, true, false, 0.0))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java index 755f7608cd9..4ca12e4ab53 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTester.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.ClusterMetricSnapshot; +import com.yahoo.vespa.hosted.provision.autoscale.Load; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; @@ -37,7 +38,6 @@ import java.util.stream.Collectors; public class AutoscalingMaintainerTester { private final ProvisioningTester provisioningTester; - private final MetricsDb metricsDb; private final AutoscalingMaintainer maintainer; private final MockDeployer deployer; @@ -49,9 +49,7 @@ public class AutoscalingMaintainerTester { Map<ApplicationId, MockDeployer.ApplicationContext> apps = Arrays.stream(appContexts) .collect(Collectors.toMap(c -> c.id(), c -> c)); deployer = new MockDeployer(provisioningTester.provisioner(), provisioningTester.clock(), apps); - metricsDb = MetricsDb.createTestInstance(provisioningTester.nodeRepository()); maintainer = new AutoscalingMaintainer(provisioningTester.nodeRepository(), - metricsDb, deployer, new TestMetric(), Duration.ofMinutes(1)); @@ -63,7 +61,6 @@ public class AutoscalingMaintainerTester { public ManualClock clock() { return provisioningTester.clock(); } public MockDeployer deployer() { return deployer; } public AutoscalingMaintainer maintainer() { return maintainer; } - public MetricsDb nodeMetricsDb() { return metricsDb; } public static ApplicationId makeApplicationId(String name) { return ProvisioningTester.applicationId(name); } public static ClusterSpec containerClusterSpec() { return ProvisioningTester.containerClusterSpec(); } @@ -76,14 +73,13 @@ public class AutoscalingMaintainerTester { NodeList nodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId); for (int i = 0; i < count; i++) { for (Node node : nodes) - metricsDb.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(clock().instant(), - cpu, - mem, - disk, - generation, - true, - true, - 0.0)))); + nodeRepository().metricsDb().addNodeMetrics(List.of(new Pair<>(node.hostname(), + new NodeMetricSnapshot(clock().instant(), + new Load(cpu, mem, disk), + generation, + true, + true, + 0.0)))); } } @@ -94,7 +90,8 @@ public class AutoscalingMaintainerTester { IntFunction<Double> queryRate) { Instant time = clock().instant(); for (int i = 0; i < measurements; i++) { - metricsDb.addClusterMetrics(application, Map.of(cluster, new ClusterMetricSnapshot(time, queryRate.apply(i), 0.0))); + nodeRepository().metricsDb().addClusterMetrics(application, + Map.of(cluster, new ClusterMetricSnapshot(time, queryRate.apply(i), 0.0))); time = time.plus(Duration.ofMinutes(5)); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index ece3f180870..7e1def4b754 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; @@ -70,6 +71,7 @@ public class CapacityCheckerTester { new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), new InMemoryFlagSource(), + new MemoryMetricsDb(clock), true, 0, 1000); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index 5af787092d5..5b2f7ce91e8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -39,15 +39,13 @@ public class NodeMetricsDbMaintainerTest { OrchestratorMock orchestrator = new OrchestratorMock(); MockHttpClient httpClient = new MockHttpClient(); MetricsV2MetricsFetcher fetcher = new MetricsV2MetricsFetcher(tester.nodeRepository(), orchestrator, httpClient); - MetricsDb db = MetricsDb.createTestInstance(tester.nodeRepository()); NodeMetricsDbMaintainer maintainer = new NodeMetricsDbMaintainer(tester.nodeRepository(), fetcher, - db, Duration.ofHours(1), new TestMetric()); assertTrue(maintainer.maintain()); - List<NodeTimeseries> timeseriesList = db.getNodeTimeseries(Duration.ofDays(1), - Set.of("host-1.yahoo.com", "host-2.yahoo.com")); + List<NodeTimeseries> timeseriesList = tester.nodeRepository().metricsDb().getNodeTimeseries(Duration.ofDays(1), + Set.of("host-1.yahoo.com", "host-2.yahoo.com")); assertEquals(2, timeseriesList.size()); List<NodeMetricSnapshot> allSnapshots = timeseriesList.stream() .flatMap(timeseries -> timeseries.asList().stream()) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 10851252c98..7ab21946379 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -17,10 +17,8 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.ClusterModel; +import com.yahoo.vespa.hosted.provision.autoscale.Load; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; -import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; -import com.yahoo.vespa.hosted.provision.autoscale.Resource; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; @@ -51,8 +49,6 @@ public class ScalingSuggestionsMaintainerTest { ApplicationId app2 = ProvisioningTester.applicationId("app2"); ClusterSpec cluster2 = ProvisioningTester.contentClusterSpec(); - MetricsDb metricsDb = MetricsDb.createTestInstance(tester.nodeRepository()); - tester.makeReadyNodes(20, "flt", NodeType.host, 8); tester.activateTenantHosts(); @@ -64,11 +60,10 @@ public class ScalingSuggestionsMaintainerTest { false, true)); tester.clock().advance(Duration.ofHours(13)); - addMeasurements(0.90f, 0.90f, 0.90f, 0, 500, app1, tester.nodeRepository(), metricsDb); - addMeasurements(0.99f, 0.99f, 0.99f, 0, 500, app2, tester.nodeRepository(), metricsDb); + addMeasurements(0.90f, 0.90f, 0.90f, 0, 500, app1, tester.nodeRepository()); + addMeasurements(0.99f, 0.99f, 0.99f, 0, 500, app2, tester.nodeRepository()); ScalingSuggestionsMaintainer maintainer = new ScalingSuggestionsMaintainer(tester.nodeRepository(), - metricsDb, Duration.ofMinutes(1), new TestMetric()); maintainer.maintain(); @@ -80,14 +75,14 @@ public class ScalingSuggestionsMaintainerTest { // Utilization goes way down tester.clock().advance(Duration.ofHours(13)); - addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository(), metricsDb); + addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Suggestion stays at the peak value observed", "12 nodes with [vcpu: 6.0, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps]", suggestionOf(app1, cluster1, tester).get().resources().toString()); // Utilization is still way down and a week has passed tester.clock().advance(Duration.ofDays(7)); - addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository(), metricsDb); + addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Peak suggestion has been outdated", "5 nodes with [vcpu: 1.8, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps]", @@ -95,13 +90,13 @@ public class ScalingSuggestionsMaintainerTest { assertTrue(shouldSuggest(app1, cluster1, tester)); tester.clock().advance(Duration.ofDays(3)); - addMeasurements(0.7f, 0.7f, 0.7f, 0, 500, app1, tester.nodeRepository(), metricsDb); + addMeasurements(0.7f, 0.7f, 0.7f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().resources(); tester.deploy(app1, cluster1, Capacity.from(suggested, suggested, false, true)); tester.clock().advance(Duration.ofDays(2)); addMeasurements(0.2f, 0.7f, 0.6f, - 0, 500, app1, tester.nodeRepository(), metricsDb); + 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Suggestion is to keep the current allocation", suggested, @@ -119,19 +114,19 @@ public class ScalingSuggestionsMaintainerTest { .shouldSuggestResources(currentResources); } - public void addMeasurements(float cpu, float memory, float disk, int generation, int count, ApplicationId applicationId, - NodeRepository nodeRepository, MetricsDb db) { + public void addMeasurements(float cpu, float memory, float disk, int generation, int count, + ApplicationId applicationId, + NodeRepository nodeRepository) { NodeList nodes = nodeRepository.nodes().list(Node.State.active).owner(applicationId); for (int i = 0; i < count; i++) { for (Node node : nodes) - db.addNodeMetrics(List.of(new Pair<>(node.hostname(), new NodeMetricSnapshot(nodeRepository.clock().instant(), - cpu, - memory, - disk, - generation, - true, - true, - 0.0)))); + nodeRepository.metricsDb().addNodeMetrics(List.of(new Pair<>(node.hostname(), + new NodeMetricSnapshot(nodeRepository.clock().instant(), + new Load(cpu, memory, disk), + generation, + true, + true, + 0.0)))); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index bb3788ba186..2790c8dc9c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; @@ -254,14 +255,16 @@ public class SpareCapacityMaintainerTest { private SpareCapacityMaintainerTester(int maxIterations) { NodeFlavors flavors = new NodeFlavors(new FlavorConfigBuilder().build()); + ManualClock clock = new ManualClock(); nodeRepository = new NodeRepository(flavors, new EmptyProvisionServiceProvider(), new MockCurator(), - new ManualClock(), + clock, new Zone(Environment.prod, RegionName.from("us-east-3")), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), new InMemoryFlagSource(), + new MemoryMetricsDb(clock), true, 1, 1000); deployer = new MockDeployer(nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index cbfaaeaf61c..73eec4b2988 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -62,7 +62,6 @@ public class InfraDeployerImplTest { private final NodeRepositoryTester tester = new NodeRepositoryTester(); private final NodeRepository nodeRepository = tester.nodeRepository(); private final Provisioner provisioner = spy(new NodeRepositoryProvisioner(nodeRepository, - new MemoryMetricsDb(nodeRepository), Zone.defaultZone(), new EmptyProvisionServiceProvider(), new InMemoryFlagSource())); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 00d87bc448e..df101c416c1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -112,12 +112,12 @@ public class ProvisioningTester { nameResolver, containerImage, flagSource, + new MemoryMetricsDb(clock), true, spareCount, 1000); this.orchestrator = orchestrator; this.provisioner = new NodeRepositoryProvisioner(nodeRepository, - new MemoryMetricsDb(nodeRepository), zone, provisionServiceProvider, flagSource); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index 7417318c572..57dcb5f3069 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -81,9 +81,10 @@ class ClusterApiImpl implements ClusterApi { servicesDownAndNotInGroup = servicesNotInGroup.stream().filter(this::serviceEffectivelyDown).collect(Collectors.toSet()); int serviceInstances = serviceCluster.serviceInstances().size(); - if (serviceCluster.isConfigServerLike() && serviceInstances < numberOfConfigServers) { + if ((serviceCluster.isConfigServerLike() || serviceCluster.isConfigServerHostLike()) && + serviceInstances < numberOfConfigServers) { missingServices = numberOfConfigServers - serviceInstances; - descriptionOfMissingServices = missingServices + " missing config server" + (missingServices > 1 ? "s" : ""); + descriptionOfMissingServices = missingServices + " missing " + serviceCluster.nodeDescription(missingServices > 1); } else { missingServices = 0; descriptionOfMissingServices = "NA"; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index ff1c56f6b2f..bd0075b403c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.policy; +import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; @@ -150,6 +151,10 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { } if (clusterApi.serviceType().equals(ServiceType.HOST_ADMIN)) { + if (Set.of(ClusterId.CONFIG_SERVER_HOST, ClusterId.CONTROLLER_HOST).contains(clusterApi.clusterId())) { + return ConcurrentSuspensionLimitForCluster.ONE_NODE; + } + return ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT; } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 412a08a8305..6d7c79176fb 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -1,17 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.model; +import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; -import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.OrchestratorUtil; @@ -20,6 +18,9 @@ import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.duper.ConfigServerHostApplication; +import com.yahoo.vespa.service.duper.InfraApplication; import org.junit.Test; import java.time.Duration; @@ -131,6 +132,38 @@ public class ClusterApiImplTest { } @Test + public void testCfghost1SuspensionFailsWithMissingCfghost3() { + ClusterApiImpl clusterApi = makeConfigClusterApi( + ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, + new ConfigServerHostApplication(), + ServiceStatus.UP, + ServiceStatus.UP); + + HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource); + + try { + policy.verifyGroupGoingDownIsFine(clusterApi); + fail(); + } catch (HostStateChangeDeniedException e) { + assertThat(e.getMessage(), + containsString("Changing the state of cfg1 would violate enough-services-up: " + + "Suspension of service with type 'hostadmin' not allowed: 33% are suspended already. " + + "Services down on resumed hosts: [1 missing config server host].")); + } + + flagSource.withBooleanFlag(Flags.GROUP_SUSPENSION.id(), true); + + try { + policy.verifyGroupGoingDownIsFine(clusterApi); + fail(); + } catch (HostStateChangeDeniedException e) { + assertThat(e.getMessage(), + containsString("Suspension of service with type 'hostadmin' not allowed: 33% are suspended already. " + + "Services down on resumed hosts: [1 missing config server host].")); + } + } + + @Test public void testCfg1SuspendsIfDownWithMissingCfg3() throws HostStateChangeDeniedException { ClusterApiImpl clusterApi = makeCfg1ClusterApi(ServiceStatus.DOWN, ServiceStatus.UP); @@ -140,6 +173,19 @@ public class ClusterApiImplTest { } @Test + public void testCfghost1SuspendsIfDownWithMissingCfghost3() throws HostStateChangeDeniedException { + ClusterApiImpl clusterApi = makeConfigClusterApi( + ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, + new ConfigServerHostApplication(), + ServiceStatus.DOWN, + ServiceStatus.UP); + + HostedVespaClusterPolicy policy = new HostedVespaClusterPolicy(flagSource); + + policy.verifyGroupGoingDownIsFine(clusterApi); + } + + @Test public void testSingleConfigServerCanSuspend() { for (var status : EnumSet.of(ServiceStatus.UP, ServiceStatus.DOWN)) { var clusterApi = makeConfigClusterApi(1, status); @@ -265,6 +311,11 @@ public class ClusterApiImplTest { } private ClusterApiImpl makeConfigClusterApi(int clusterSize, ServiceStatus first, ServiceStatus... rest) { + return makeConfigClusterApi(clusterSize, new ConfigServerApplication(), first, rest); + } + + private ClusterApiImpl makeConfigClusterApi(int clusterSize, InfraApplication infraApplication, + ServiceStatus first, ServiceStatus... rest) { var serviceStatusList = new ArrayList<ServiceStatus>(); serviceStatusList.add(first); serviceStatusList.addAll(List.of(rest)); @@ -275,9 +326,10 @@ public class ClusterApiImplTest { for (int i = 0; i < hostnames.size(); i++) { instances.add(modelUtils.createServiceInstance("cs" + i + 1, hostnames.get(i), serviceStatusList.get(i))); } + ServiceCluster serviceCluster = modelUtils.createServiceCluster( - ClusterId.CONFIG_SERVER.s(), - ServiceType.CONFIG_SERVER, + infraApplication.getClusterId().s(), + infraApplication.getServiceType(), instances ); for (var instance : instances) { @@ -287,8 +339,8 @@ public class ClusterApiImplTest { Set<ServiceCluster> serviceClusterSet = Set.of(serviceCluster); ApplicationInstance application = new ApplicationInstance( - TenantId.HOSTED_VESPA, - ApplicationInstanceId.CONFIG_SERVER, + infraApplication.getTenantId(), + infraApplication.getApplicationInstanceId(Zone.defaultZone()), serviceClusterSet); serviceCluster.setApplicationInstance(application); diff --git a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp index 3dc1cf6d27e..98b2bfe5c90 100644 --- a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp +++ b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp @@ -110,22 +110,24 @@ TEST("Test GrowStrategy consistency") { } TEST("DictionaryConfig") { - using Ordering = DictionaryConfig::Ordering; - EXPECT_EQUAL(Ordering::ORDERED, DictionaryConfig().getOrdering()); - EXPECT_EQUAL(Ordering::ORDERED, DictionaryConfig(Ordering::ORDERED).getOrdering()); - EXPECT_EQUAL(Ordering::UNORDERED, DictionaryConfig(Ordering::UNORDERED).getOrdering()); - EXPECT_EQUAL(DictionaryConfig(Ordering::ORDERED), DictionaryConfig(Ordering::ORDERED)); - EXPECT_EQUAL(DictionaryConfig(Ordering::UNORDERED), DictionaryConfig(Ordering::UNORDERED)); - EXPECT_NOT_EQUAL(DictionaryConfig(Ordering::UNORDERED), DictionaryConfig(Ordering::ORDERED)); - EXPECT_NOT_EQUAL(DictionaryConfig(Ordering::ORDERED), DictionaryConfig(Ordering::UNORDERED)); - EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) == - Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED))); - EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) == - Config().set_dictionary_config(DictionaryConfig(Ordering::ORDERED))); - EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) != - Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED))); - EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) != - Config().set_dictionary_config(DictionaryConfig(Ordering::ORDERED))); + using Type = DictionaryConfig::Type; + EXPECT_EQUAL(Type::BTREE, DictionaryConfig().getType()); + EXPECT_EQUAL(Type::BTREE, DictionaryConfig(Type::BTREE).getType()); + EXPECT_EQUAL(Type::HASH, DictionaryConfig(Type::HASH).getType()); + EXPECT_EQUAL(Type::BTREE_AND_HASH, DictionaryConfig(Type::BTREE_AND_HASH).getType()); + EXPECT_EQUAL(DictionaryConfig(Type::BTREE), DictionaryConfig(Type::BTREE)); + EXPECT_EQUAL(DictionaryConfig(Type::HASH), DictionaryConfig(Type::HASH)); + EXPECT_EQUAL(DictionaryConfig(Type::BTREE_AND_HASH), DictionaryConfig(Type::BTREE_AND_HASH)); + EXPECT_NOT_EQUAL(DictionaryConfig(Type::HASH), DictionaryConfig(Type::BTREE)); + EXPECT_NOT_EQUAL(DictionaryConfig(Type::BTREE), DictionaryConfig(Type::HASH)); + EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) == + Config().set_dictionary_config(DictionaryConfig(Type::HASH))); + EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) == + Config().set_dictionary_config(DictionaryConfig(Type::BTREE))); + EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) != + Config().set_dictionary_config(DictionaryConfig(Type::HASH))); + EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) != + Config().set_dictionary_config(DictionaryConfig(Type::BTREE))); } diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp b/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp index 00b6ae2710f..a6a0255f96d 100644 --- a/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp +++ b/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp @@ -8,11 +8,13 @@ namespace search { std::ostream& operator<<(std::ostream& os, const DictionaryConfig & cfg) { - switch(cfg.getOrdering()) { - case DictionaryConfig::Ordering::ORDERED: - return os << "ORDERED"; - case DictionaryConfig::Ordering::UNORDERED: - return os << "UNORDERED"; + switch(cfg.getType()) { + case DictionaryConfig::Type::BTREE: + return os << "BTREE"; + case DictionaryConfig::Type::HASH: + return os << "HASH"; + case DictionaryConfig::Type::BTREE_AND_HASH: + return os << "BTREE_AND_HASH"; } assert(false); } diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.h b/searchcommon/src/vespa/searchcommon/common/dictionary_config.h index 06c88d88670..c35f7eaafef 100644 --- a/searchcommon/src/vespa/searchcommon/common/dictionary_config.h +++ b/searchcommon/src/vespa/searchcommon/common/dictionary_config.h @@ -11,13 +11,13 @@ namespace search { */ class DictionaryConfig { public: - enum class Ordering { ORDERED, UNORDERED }; - DictionaryConfig() noexcept : _ordering(Ordering::ORDERED) {} - DictionaryConfig(Ordering ordering) noexcept : _ordering(ordering) {} - Ordering getOrdering() const { return _ordering; } - bool operator == (const DictionaryConfig & b) const { return _ordering == b._ordering; } + enum class Type { BTREE, HASH, BTREE_AND_HASH }; + DictionaryConfig() noexcept : _type(Type::BTREE) {} + DictionaryConfig(Type ordering) noexcept : _type(ordering) {} + Type getType() const { return _type; } + bool operator == (const DictionaryConfig & b) const { return _type == b._type; } private: - Ordering _ordering; + Type _type; }; std::ostream& operator<<(std::ostream& os, const DictionaryConfig & cfg); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp index 69bd743021a..042cd70f7e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp @@ -45,8 +45,8 @@ isSameDocument(const search::DocumentMetaData & a, const search::DocumentMetaDat void CompactionJob::failOperation() { - _executedCount.fetch_add(1, std::memory_order_relaxed); - _scanItr.reset(); + IncOnDestruct countGuard(_executedCount); + _master.execute(makeLambdaTask([this] { _scanItr.reset(); })); } bool @@ -62,7 +62,7 @@ CompactionJob::scanDocuments(const LidUsageStats &stats) assert(bucket.getBucketId() == meta.bucketId); using DoneContext = vespalib::KeepAlive<std::pair<IDestructorCallback::SP, IDestructorCallback::SP>>; moveDocument(meta, std::make_shared<DoneContext>(std::make_pair(std::move(opsTracker), std::move(onDone)))); - }, [this](const Bucket &) { _master.execute(makeLambdaTask([this] { failOperation(); } )); }); + }, [this](const Bucket &) { failOperation(); }); _startedCount.fetch_add(1, std::memory_order_relaxed); _bucketExecutor.execute(metaBucket, std::move(bucketTask)); diff --git a/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp b/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp index 3690533eef9..d999a6f37a2 100644 --- a/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp +++ b/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp @@ -27,7 +27,7 @@ using NodeAllocator = TreeType::NodeAllocatorType; TEST("requireThatNumericLessIsWorking") { - NumericEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10); EnumIndex e2 = es.insert(30); auto cmp1 = es.make_comparator(); @@ -41,7 +41,7 @@ TEST("requireThatNumericLessIsWorking") TEST("requireThatNumericEqualIsWorking") { - NumericEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10); EnumIndex e2 = es.insert(30); auto cmp1 = es.make_comparator(); @@ -56,7 +56,7 @@ TEST("requireThatNumericEqualIsWorking") TEST("requireThatFloatLessIsWorking") { - FloatEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + FloatEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10.5); EnumIndex e2 = es.insert(30.5); EnumIndex e3 = es.insert(std::numeric_limits<float>::quiet_NaN()); @@ -74,7 +74,7 @@ TEST("requireThatFloatLessIsWorking") TEST("requireThatFloatEqualIsWorking") { - FloatEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + FloatEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10.5); EnumIndex e2 = es.insert(30.5); EnumIndex e3 = es.insert(std::numeric_limits<float>::quiet_NaN()); @@ -93,7 +93,7 @@ TEST("requireThatFloatEqualIsWorking") TEST("requireThatStringLessIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); @@ -110,7 +110,7 @@ TEST("requireThatStringLessIsWorking") TEST("requireThatStringEqualIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); @@ -127,7 +127,7 @@ TEST("requireThatStringEqualIsWorking") TEST("requireThatComparatorWithTreeIsWorking") { - NumericEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore es(false, DictionaryConfig::Type::BTREE); vespalib::GenerationHandler g; TreeType t; NodeAllocator m; @@ -152,7 +152,7 @@ TEST("requireThatComparatorWithTreeIsWorking") TEST("requireThatFoldedLessIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); @@ -172,7 +172,7 @@ TEST("requireThatFoldedLessIsWorking") TEST("requireThatFoldedEqualIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 85c12acb57d..fb514063e73 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -6,7 +6,8 @@ #include <vespa/log/log.h> LOG_SETUP("enumstore_test"); -using Ordering = search::DictionaryConfig::Ordering; +using Type = search::DictionaryConfig::Type; +using vespalib::datastore::EntryRef; namespace search { @@ -18,42 +19,42 @@ using StringEnumStore = EnumStoreT<const char*>; struct OrderedDoubleEnumStore { using EnumStoreType = DoubleEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedDoubleEnumStore { using EnumStoreType = DoubleEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; struct OrderedFloatEnumStore { using EnumStoreType = FloatEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedFloatEnumStore { using EnumStoreType = FloatEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; struct OrderedNumericEnumStore { using EnumStoreType = NumericEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedNumericEnumStore { using EnumStoreType = NumericEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; struct OrderedStringEnumStore { using EnumStoreType = StringEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedStringEnumStore { using EnumStoreType = StringEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; using StringVector = std::vector<std::string>; @@ -105,7 +106,7 @@ public: using EnumStoreType = typename EnumStoreTypeAndOrdering::EnumStoreType; EnumStoreType es; FloatEnumStoreTest() - : es(false, EnumStoreTypeAndOrdering::ordering) + : es(false, EnumStoreTypeAndOrdering::type) {} }; @@ -149,7 +150,7 @@ TYPED_TEST(FloatEnumStoreTest, numbers_can_be_inserted_and_retrieved) TEST(EnumStoreTest, test_find_folded_on_string_enum_store) { - StringEnumStore ses(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore ses(false, DictionaryConfig::Type::BTREE); std::vector<EnumIndex> indices; std::vector<std::string> unique({"", "one", "two", "TWO", "Two", "three"}); for (std::string &str : unique) { @@ -200,7 +201,7 @@ public: void StringEnumStoreTest::testInsert(bool hasPostings) { - StringEnumStore ses(hasPostings, DictionaryConfig::Ordering::ORDERED); + StringEnumStore ses(hasPostings, DictionaryConfig::Type::BTREE); std::vector<EnumIndex> indices; std::vector<std::string> unique; @@ -250,7 +251,7 @@ TEST_F(StringEnumStoreTest, test_insert_on_store_with_posting_lists) TEST(EnumStoreTest, test_hold_lists_and_generation) { - StringEnumStore ses(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore ses(false, DictionaryConfig::Type::BTREE); StringVector uniques; generation_t sesGen = 0u; uniques.reserve(100); @@ -327,7 +328,7 @@ dec_ref_count(NumericEnumStore& store, NumericEnumStore::Index idx) TEST(EnumStoreTest, address_space_usage_is_reported) { const size_t ADDRESS_LIMIT = 4290772994; // Max allocated elements in un-allocated buffers + allocated elements in allocated buffers. - NumericEnumStore store(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore store(false, DictionaryConfig::Type::BTREE); using vespalib::AddressSpace; EXPECT_EQ(AddressSpace(1, 1, ADDRESS_LIMIT), store.get_address_space_usage()); @@ -349,7 +350,7 @@ public: EnumIndex i5; BatchUpdaterTest() - : store(false, DictionaryConfig::Ordering::ORDERED), + : store(false, DictionaryConfig::Type::BTREE), i3(), i5() { @@ -457,7 +458,7 @@ public: using Values = LoaderTestValues<EnumStoreType>; LoaderTest() - : store(true, EnumStoreTypeAndOrdering::ordering) + : store(true, EnumStoreTypeAndOrdering::type) {} void load_values(enumstore::EnumeratedLoaderBase& loader) const { @@ -471,7 +472,8 @@ public: } void set_ref_count(size_t values_idx, uint32_t ref_count, enumstore::EnumeratedPostingsLoader& loader) const { - EnumIndex idx = find_index(values_idx); + assert(values_idx < loader.get_enum_indexes().size()); + EnumIndex idx = loader.get_enum_indexes()[values_idx]; loader.set_ref_count(idx, ref_count); } @@ -493,10 +495,11 @@ public: } void expect_posting_idx(size_t values_idx, uint32_t exp_posting_idx) const { - auto cmp = store.make_comparator(); - auto itr = store.get_posting_dictionary().find(find_index(values_idx), cmp); - ASSERT_TRUE(itr.valid()); - EXPECT_EQ(exp_posting_idx, itr.getData()); + auto cmp = store.make_comparator(Values::values[values_idx]); + auto &dict = store.get_dictionary(); + auto find_result = dict.find_posting_list(cmp, dict.get_frozen_root()); + ASSERT_TRUE(find_result.first.valid()); + EXPECT_EQ(exp_posting_idx, find_result.second.ref()); } }; @@ -519,6 +522,8 @@ TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_loader) loader.get_enums_histogram()[1] = 2; loader.get_enums_histogram()[3] = 4; loader.set_ref_counts(); + loader.build_dictionary(); + loader.free_unused_values(); this->expect_values_in_store(); } @@ -530,6 +535,8 @@ TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_postings_loader) this->set_ref_count(0, 1, loader); this->set_ref_count(1, 2, loader); this->set_ref_count(3, 4, loader); + loader.initialize_empty_posting_indexes(); + loader.build_dictionary(); loader.free_unused_values(); this->expect_values_in_store(); @@ -548,6 +555,7 @@ TYPED_TEST(LoaderTest, store_is_instantiated_with_non_enumerated_loader) loader.build_dictionary(); this->expect_values_in_store(); + this->store.freeze_dictionary(); this->expect_posting_idx(0, 100); this->expect_posting_idx(1, 101); @@ -556,6 +564,129 @@ TYPED_TEST(LoaderTest, store_is_instantiated_with_non_enumerated_loader) #pragma GCC diagnostic pop +template <typename EnumStoreTypeAndOrdering> +class EnumStoreDictionaryTest : public ::testing::Test { +public: + using EnumStoreType = typename EnumStoreTypeAndOrdering::EnumStoreType; + using EntryType = typename EnumStoreType::EntryType; + EnumStoreType store; + + EnumStoreDictionaryTest() + : store(true, EnumStoreTypeAndOrdering::type) + {} + + // Reuse test values from LoaderTest + const std::vector<EntryType>& values() const noexcept { return LoaderTestValues<EnumStoreType>::values; } + + typename EnumStoreType::ComparatorType make_bound_comparator(int value_idx) { return store.make_comparator(values()[value_idx]); } + + void update_posting_idx(EnumIndex enum_idx, EntryRef old_posting_idx, EntryRef new_posting_idx); + EnumIndex insert_value(size_t value_idx); + static EntryRef fake_pidx() { return EntryRef(42); } +}; + +template <typename EnumStoreTypeAndOrdering> +void +EnumStoreDictionaryTest<EnumStoreTypeAndOrdering>::update_posting_idx(EnumIndex enum_idx, EntryRef old_posting_idx, EntryRef new_posting_idx) +{ + auto& dict = store.get_dictionary(); + EntryRef old_posting_idx_check; + dict.update_posting_list(enum_idx, store.make_comparator(), [&old_posting_idx_check, new_posting_idx](EntryRef posting_idx) noexcept -> EntryRef { old_posting_idx_check = posting_idx; return new_posting_idx; }); + EXPECT_EQ(old_posting_idx, old_posting_idx_check); +} + +template <typename EnumStoreTypeAndOrdering> +EnumIndex +EnumStoreDictionaryTest<EnumStoreTypeAndOrdering>::insert_value(size_t value_idx) +{ + assert(value_idx < values().size()); + auto enum_idx = store.insert(values()[value_idx]); + EXPECT_TRUE(enum_idx.valid()); + return enum_idx; +} + +// Disable warnings emitted by gtest generated files when using typed tests +#pragma GCC diagnostic push +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif + +using EnumStoreDictionaryTestTypes = ::testing::Types<OrderedNumericEnumStore, UnorderedNumericEnumStore>; +VESPA_GTEST_TYPED_TEST_SUITE(EnumStoreDictionaryTest, EnumStoreDictionaryTestTypes); + +TYPED_TEST(EnumStoreDictionaryTest, find_frozen_index_works) +{ + auto value_0_idx = this->insert_value(0); + this->update_posting_idx(value_0_idx, EntryRef(), this->fake_pidx()); + auto& dict = this->store.get_dictionary(); + EnumIndex idx; + if (TypeParam::type == Type::BTREE) { + EXPECT_FALSE(dict.find_frozen_index(this->make_bound_comparator(0), idx)); + } else { + EXPECT_TRUE(dict.find_frozen_index(this->make_bound_comparator(0), idx)); + EXPECT_EQ(value_0_idx, idx); + } + EXPECT_FALSE(dict.find_frozen_index(this->make_bound_comparator(1), idx)); + this->store.freeze_dictionary(); + idx = EnumIndex(); + EXPECT_TRUE(dict.find_frozen_index(this->make_bound_comparator(0), idx)); + EXPECT_EQ(value_0_idx, idx); + EXPECT_FALSE(dict.find_frozen_index(this->make_bound_comparator(1), idx)); + this->update_posting_idx(value_0_idx, this->fake_pidx(), EntryRef()); +} + +TYPED_TEST(EnumStoreDictionaryTest, find_posting_list_works) +{ + auto value_0_idx = this->insert_value(0); + this->update_posting_idx(value_0_idx, EntryRef(), this->fake_pidx()); + auto& dict = this->store.get_dictionary(); + auto root = dict.get_frozen_root(); + auto find_result = dict.find_posting_list(this->make_bound_comparator(0), root); + if (TypeParam::type == Type::BTREE) { + EXPECT_FALSE(find_result.first.valid()); + EXPECT_FALSE(find_result.second.valid()); + } else { + EXPECT_EQ(value_0_idx, find_result.first); + EXPECT_EQ(this->fake_pidx(), find_result.second); + } + find_result = dict.find_posting_list(this->make_bound_comparator(1), root); + EXPECT_FALSE(find_result.first.valid()); + this->store.freeze_dictionary(); + root = dict.get_frozen_root(); + find_result = dict.find_posting_list(this->make_bound_comparator(0), root); + EXPECT_EQ(value_0_idx, find_result.first); + EXPECT_EQ(this->fake_pidx(), find_result.second); + find_result = dict.find_posting_list(this->make_bound_comparator(1), root); + EXPECT_FALSE(find_result.first.valid()); + this->update_posting_idx(value_0_idx, this->fake_pidx(), EntryRef()); +} + +TYPED_TEST(EnumStoreDictionaryTest, normalize_posting_lists_works) +{ + auto value_0_idx = this->insert_value(0); + this->update_posting_idx(value_0_idx, EntryRef(), this->fake_pidx()); + this->store.freeze_dictionary(); + auto& dict = this->store.get_dictionary(); + auto root = dict.get_frozen_root(); + auto find_result = dict.find_posting_list(this->make_bound_comparator(0), root); + EXPECT_EQ(value_0_idx, find_result.first); + EXPECT_EQ(this->fake_pidx(), find_result.second); + auto dummy = [](EntryRef posting_idx) { return posting_idx; }; + std::vector<EntryRef> saved_refs; + auto save_refs_and_clear = [&saved_refs](EntryRef posting_idx) { saved_refs.push_back(posting_idx); return EntryRef(); }; + EXPECT_FALSE(dict.normalize_posting_lists(dummy)); + EXPECT_TRUE(dict.normalize_posting_lists(save_refs_and_clear)); + EXPECT_FALSE(dict.normalize_posting_lists(save_refs_and_clear)); + EXPECT_EQ((std::vector<EntryRef>{ this->fake_pidx(), EntryRef() }), saved_refs); + this->store.freeze_dictionary(); + root = dict.get_frozen_root(); + find_result = dict.find_posting_list(this->make_bound_comparator(0), root); + EXPECT_EQ(value_0_idx, find_result.first); + EXPECT_EQ(EntryRef(), find_result.second); +} + +#pragma GCC diagnostic pop + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index bde75edcd03..d6238d848b4 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -446,18 +446,18 @@ PostingListAttributeTest::checkPostingList(const VectorType & vec, const std::ve const RangeGenerator & range) { const typename VectorType::EnumStore & enumStore = vec.getEnumStore(); - const typename VectorType::Dictionary & dict = enumStore.get_posting_dictionary(); + auto& dict = enumStore.get_dictionary(); const typename VectorType::PostingList & postingList = vec.getPostingList(); for (size_t i = 0; i < values.size(); ++i) { const uint32_t docBegin = range.getBegin(i); const uint32_t docEnd = range.getEnd(i); - auto itr = dict.find(enumstore::Index(), enumStore.make_comparator(values[i])); - ASSERT_TRUE(itr.valid()); + auto find_result = dict.find_posting_list(enumStore.make_comparator(values[i]), dict.get_frozen_root()); + ASSERT_TRUE(find_result.first.valid()); typename VectorType::PostingList::Iterator postings; - postings = postingList.begin(vespalib::datastore::EntryRef(itr.getData())); + postings = postingList.begin(find_result.second); uint32_t doc = docBegin; uint32_t numHits(0); @@ -669,14 +669,13 @@ void PostingListAttributeTest::checkPostingList(AttributeType & vec, ValueType value, DocSet expected) { const typename AttributeType::EnumStore & enumStore = vec.getEnumStore(); - const typename AttributeType::Dictionary & dict = enumStore.get_posting_dictionary(); + auto& dict = enumStore.get_dictionary(); const typename AttributeType::PostingList & postingList = vec.getPostingList(); - auto itr = dict.find(typename AttributeType::EnumIndex(), - vec.getEnumStore().make_comparator(value)); - ASSERT_TRUE(itr.valid()); + auto find_result = dict.find_posting_list(vec.getEnumStore().make_comparator(value), dict.get_frozen_root()); + ASSERT_TRUE(find_result.first.valid()); typename AttributeType::PostingList::Iterator postings; - postings = postingList.begin(vespalib::datastore::EntryRef(itr.getData())); + postings = postingList.begin(find_result.second); DocSet::iterator docBegin = expected.begin(); DocSet::iterator docEnd = expected.end(); @@ -690,10 +689,9 @@ template <typename AttributeType, typename ValueType> void PostingListAttributeTest::checkNonExistantPostingList(AttributeType & vec, ValueType value) { - const typename AttributeType::Dictionary & dict = vec.getEnumStore().get_posting_dictionary(); - auto itr = dict.find(typename AttributeType::EnumIndex(), - vec.getEnumStore().make_comparator(value)); - EXPECT_TRUE(!itr.valid()); + auto& dict = vec.getEnumStore().get_dictionary(); + auto find_result = dict.find_posting_list(vec.getEnumStore().make_comparator(value), dict.get_frozen_root()); + EXPECT_TRUE(!find_result.first.valid()); } template <typename AttributeType, typename ValueType> diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 145a021801d..2e946c7d34b 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -50,20 +50,22 @@ getCollectionTypeMap() static DataTypeMap _dataTypeMap = getDataTypeMap(); static CollectionTypeMap _collectionTypeMap = getCollectionTypeMap(); -DictionaryConfig::Ordering -convert(AttributesConfig::Attribute::Dictionary::Ordering ordering_cfg) { - switch (ordering_cfg) { - case AttributesConfig::Attribute::Dictionary::Ordering::ORDERED: - return DictionaryConfig::Ordering::ORDERED; - case AttributesConfig::Attribute::Dictionary::Ordering::UNORDERED: - return DictionaryConfig::Ordering::UNORDERED; +DictionaryConfig::Type +convert(AttributesConfig::Attribute::Dictionary::Type type_cfg) { + switch (type_cfg) { + case AttributesConfig::Attribute::Dictionary::Type::BTREE: + return DictionaryConfig::Type::BTREE; + case AttributesConfig::Attribute::Dictionary::Type::HASH: + return DictionaryConfig::Type::HASH; + case AttributesConfig::Attribute::Dictionary::Type::BTREE_AND_HASH: + return DictionaryConfig::Type::BTREE_AND_HASH; } assert(false); } DictionaryConfig convert_dictionary(const AttributesConfig::Attribute::Dictionary & dictionary) { - return DictionaryConfig(convert(dictionary.ordering)); + return DictionaryConfig(convert(dictionary.type)); } } diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.h b/searchlib/src/vespa/searchlib/attribute/diversity.h index 6e3c6cfff00..ff7d9b5f83d 100644 --- a/searchlib/src/vespa/searchlib/attribute/diversity.h +++ b/searchlib/src/vespa/searchlib/attribute/diversity.h @@ -123,4 +123,22 @@ void diversify(bool forward, const DictItr &lower, const DictItr &upper, const P } } +template <typename PostingStore, typename Result> +void diversify_single(vespalib::datastore::EntryRef posting_idx, const PostingStore &posting, size_t wanted_hits, + const IAttributeVector &diversity_attr, size_t max_per_group, + size_t cutoff_max_groups, bool cutoff_strict, + Result &result, std::vector<size_t> &fragments) +{ + auto filter = DiversityFilter::create(diversity_attr, wanted_hits, max_per_group, cutoff_max_groups, cutoff_strict); + DiversityRecorder<Result> recorder(*filter, result); + using DataType = typename PostingStore::DataType; + using KeyDataType = typename PostingStore::KeyDataType; + posting.foreach_frozen(posting_idx, + [&](uint32_t key, const DataType &data) + { recorder.push_back(KeyDataType(key, data)); }); + if (fragments.back() < result.size()) { + fragments.push_back(result.size()); + } +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index 1676020b417..1d0430cfb63 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -23,9 +23,9 @@ namespace search { using vespalib::btree::BTreeNode; -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::remove_unused_values(const IndexSet& unused, +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove_unused_values(const IndexSet& unused, const vespalib::datastore::EntryComparator& cmp) { if (unused.empty()) { @@ -36,39 +36,32 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::remove_unused_values(con } } -template <typename DictionaryT, typename UnorderedDictionaryT> -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare) +template <typename BTreeDictionaryT, typename HashDictionaryT> +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare) : ParentUniqueStoreDictionary(std::move(compare)), _enumStore(enumStore) { } -template <typename DictionaryT, typename UnorderedDictionaryT> -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::~EnumStoreDictionary() = default; +template <typename BTreeDictionaryT, typename HashDictionaryT> +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::~EnumStoreDictionary() = default; -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::set_ref_counts(const EnumVector& hist) -{ - _enumStore.set_ref_counts(hist, this->_dict); -} - -template <typename DictionaryT, typename UnorderedDictionaryT> -void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::free_unused_values(const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const vespalib::datastore::EntryComparator& cmp) { IndexSet unused; // find unused enums - for (auto iter = this->_dict.begin(); iter.valid(); ++iter) { + for (auto iter = this->_btree_dict.begin(); iter.valid(); ++iter) { _enumStore.free_value_if_unused(iter.getKey(), unused); } remove_unused_values(unused, cmp); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::free_unused_values(const IndexSet& to_remove, +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const IndexSet& to_remove, const vespalib::datastore::EntryComparator& cmp) { IndexSet unused; @@ -78,29 +71,29 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::free_unused_values(const remove_unused_values(unused, cmp); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::remove(const EntryComparator &comp, EntryRef ref) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove(const EntryComparator &comp, EntryRef ref) { assert(ref.valid()); - auto itr = this->_dict.lowerBound(ref, comp); + auto itr = this->_btree_dict.lowerBound(ref, comp); assert(itr.valid() && itr.getKey() == ref); - if constexpr (std::is_same_v<DictionaryT, EnumPostingTree>) { + if constexpr (std::is_same_v<BTreeDictionaryT, EnumPostingTree>) { assert(EntryRef(itr.getData()) == EntryRef()); } - this->_dict.remove(itr); - if constexpr (has_unordered_dictionary) { - auto *result = this->_unordered_dict.remove(comp, ref); + this->_btree_dict.remove(itr); + if constexpr (has_hash_dictionary) { + auto *result = this->_hash_dict.remove(comp, ref); assert(result != nullptr && result->first.load_relaxed() == ref); } } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> bool -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_index(const vespalib::datastore::EntryComparator& cmp, +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const { - auto itr = this->_dict.find(Index(), cmp); + auto itr = this->_btree_dict.find(Index(), cmp); if (!itr.valid()) { return false; } @@ -108,20 +101,20 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_index(const vespali return true; } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> bool -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_frozen_index(const vespalib::datastore::EntryComparator& cmp, +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const { - if constexpr (has_unordered_dictionary) { - auto find_result = this->_unordered_dict.find(cmp, EntryRef()); + if constexpr (has_hash_dictionary) { + auto find_result = this->_hash_dict.find(cmp, EntryRef()); if (find_result != nullptr) { idx = find_result->first.load_acquire(); return true; } return false; } - auto itr = this->_dict.getFrozenView().find(Index(), cmp); + auto itr = this->_btree_dict.getFrozenView().find(Index(), cmp); if (!itr.valid()) { return false; } @@ -129,12 +122,12 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_frozen_index(const return true; } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> std::vector<IEnumStore::EnumHandle> -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const { std::vector<IEnumStore::EnumHandle> result; - auto itr = this->_dict.getFrozenView().find(Index(), cmp); + auto itr = this->_btree_dict.getFrozenView().find(Index(), cmp); while (itr.valid() && !cmp.less(Index(), itr.getKey())) { result.push_back(itr.getKey().ref()); ++itr; @@ -142,11 +135,11 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_matching_enums(cons return result; } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> EntryRef -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::get_frozen_root() const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_frozen_root() const { - return this->_dict.getFrozenView().getRoot(); + return this->_btree_dict.getFrozenView().getRoot(); } template <> @@ -156,18 +149,18 @@ EnumStoreDictionary<EnumTree>::find_posting_list(const vespalib::datastore::Entr LOG_ABORT("should not be reached"); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> std::pair<IEnumStore::Index, EntryRef> -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const { - if constexpr (has_unordered_dictionary) { - auto find_result = this->_unordered_dict.find(cmp, EntryRef()); + if constexpr (has_hash_dictionary) { + auto find_result = this->_hash_dict.find(cmp, EntryRef()); if (find_result != nullptr) { return std::make_pair(find_result->first.load_acquire(), find_result->second.load_acquire()); } return std::make_pair(Index(), EntryRef()); } - typename DictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), this->_dict.getAllocator()); + typename BTreeDictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), this->_btree_dict.getAllocator()); itr.lower_bound(root, Index(), cmp); if (itr.valid() && !cmp.less(Index(), itr.getKey())) { return std::make_pair(itr.getKey(), EntryRef(itr.getData())); @@ -175,16 +168,16 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::find_posting_list(const return std::make_pair(Index(), EntryRef()); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::collect_folded(Index idx, EntryRef, const std::function<void(vespalib::datastore::EntryRef)>& callback) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::collect_folded(Index idx, EntryRef, const std::function<void(vespalib::datastore::EntryRef)>& callback) const { callback(idx); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> IEnumStore::Index -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::remap_index(Index idx) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remap_index(Index idx) { return idx; } @@ -196,11 +189,11 @@ EnumStoreDictionary<EnumTree>::clear_all_posting_lists(std::function<void(EntryR LOG_ABORT("should not be reached"); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::clear_all_posting_lists(std::function<void(EntryRef)> clearer) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::clear_all_posting_lists(std::function<void(EntryRef)> clearer) { - auto& dict = this->_dict; + auto& dict = this->_btree_dict; auto itr = dict.begin(); EntryRef prev; while (itr.valid()) { @@ -223,46 +216,54 @@ EnumStoreDictionary<EnumTree>::update_posting_list(Index, const vespalib::datast LOG_ABORT("should not be reached"); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) { - auto& dict = this->_dict; + auto& dict = this->_btree_dict; auto itr = dict.lowerBound(idx, cmp); assert(itr.valid() && itr.getKey() == idx); EntryRef old_posting_idx(itr.getData()); EntryRef new_posting_idx = updater(old_posting_idx); dict.thaw(itr); itr.writeData(new_posting_idx.ref()); -} - -template <typename DictionaryT, typename UnorderedDictionaryT> -void -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::sync_unordered_after_load() -{ - if constexpr (has_unordered_dictionary) { - for (auto itr = this->_dict.begin(); itr.valid(); ++itr) { - EntryRef ref(itr.getKey()); - std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); - auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); - assert(add_result.first.load_relaxed() == ref); - add_result.second.store_relaxed(EntryRef(itr.getData())); - } - } + if constexpr (has_hash_dictionary) { + auto find_result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), idx); + assert(find_result != nullptr && find_result->first.load_relaxed() == idx); + assert(find_result->second.load_relaxed() == old_posting_idx); + find_result->second.store_release(new_posting_idx); + } } template <> -EnumPostingTree & -EnumStoreDictionary<EnumTree>::get_posting_dictionary() +bool +EnumStoreDictionary<EnumTree>::normalize_posting_lists(std::function<EntryRef(EntryRef)>) { LOG_ABORT("should not be reached"); } -template <typename DictionaryT, typename UnorderedDictionaryT> -EnumPostingTree & -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::get_posting_dictionary() +template <typename BTreeDictionaryT, typename HashDictionaryT> +bool +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) { - return this->_dict; + bool changed = false; + auto& dict = this->_btree_dict; + for (auto itr = dict.begin(); itr.valid(); ++itr) { + EntryRef old_posting_idx(itr.getData()); + EntryRef new_posting_idx = normalize(old_posting_idx); + if (new_posting_idx != old_posting_idx) { + changed = true; + dict.thaw(itr); + itr.writeData(new_posting_idx.ref()); + if constexpr (has_hash_dictionary) { + auto find_result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), itr.getKey()); + assert(find_result != nullptr && find_result->first.load_relaxed() == itr.getKey()); + assert(find_result->second.load_relaxed() == old_posting_idx); + find_result->second.store_release(new_posting_idx); + } + } + } + return changed; } template <> @@ -272,11 +273,11 @@ EnumStoreDictionary<EnumTree>::get_posting_dictionary() const LOG_ABORT("should not be reached"); } -template <typename DictionaryT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename HashDictionaryT> const EnumPostingTree & -EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::get_posting_dictionary() const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_posting_dictionary() const { - return this->_dict; + return this->_btree_dict; } EnumStoreFoldedDictionary::EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<EntryComparator> folded_compare) @@ -290,19 +291,19 @@ EnumStoreFoldedDictionary::~EnumStoreFoldedDictionary() = default; UniqueStoreAddResult EnumStoreFoldedDictionary::add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) { - static_assert(!has_unordered_dictionary, "Folded Dictionary does not support unordered"); - auto it = _dict.lowerBound(EntryRef(), comp); + static_assert(!has_hash_dictionary, "Folded Dictionary does not support hash dictionary"); + auto it = _btree_dict.lowerBound(EntryRef(), comp); if (it.valid() && !comp.less(EntryRef(), it.getKey())) { // Entry already exists return UniqueStoreAddResult(it.getKey(), false); } EntryRef newRef = insertEntry(); - _dict.insert(it, newRef, EntryRef().ref()); + _btree_dict.insert(it, newRef, EntryRef().ref()); // Maybe move posting list reference from next entry ++it; if (it.valid() && EntryRef(it.getData()).valid() && !_folded_compare->less(newRef, it.getKey())) { EntryRef posting_list_ref(it.getData()); - _dict.thaw(it); + _btree_dict.thaw(it); it.writeData(EntryRef().ref()); --it; assert(it.valid() && it.getKey() == newRef); @@ -314,16 +315,16 @@ EnumStoreFoldedDictionary::add(const EntryComparator& comp, std::function<EntryR void EnumStoreFoldedDictionary::remove(const EntryComparator& comp, EntryRef ref) { - static_assert(!has_unordered_dictionary, "Folded Dictionary does not support unordered"); + static_assert(!has_hash_dictionary, "Folded Dictionary does not support hash dictionary"); assert(ref.valid()); - auto it = _dict.lowerBound(ref, comp); + auto it = _btree_dict.lowerBound(ref, comp); assert(it.valid() && it.getKey() == ref); EntryRef posting_list_ref(it.getData()); - _dict.remove(it); + _btree_dict.remove(it); // Maybe copy posting list reference to next entry if (posting_list_ref.valid()) { if (it.valid() && !EntryRef(it.getData()).valid() && !_folded_compare->less(ref, it.getKey())) { - this->_dict.thaw(it); + this->_btree_dict.thaw(it); it.writeData(posting_list_ref.ref()); } else { LOG_ABORT("Posting list not cleared for removed unique value"); @@ -334,7 +335,7 @@ EnumStoreFoldedDictionary::remove(const EntryComparator& comp, EntryRef ref) void EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const { - DictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), _dict.getAllocator()); + BTreeDictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), _btree_dict.getAllocator()); itr.lower_bound(root, idx, *_folded_compare); while (itr.valid() && !_folded_compare->less(idx, itr.getKey())) { callback(itr.getKey()); @@ -345,7 +346,7 @@ EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::f IEnumStore::Index EnumStoreFoldedDictionary::remap_index(Index idx) { - auto itr = _dict.find(idx, *_folded_compare); + auto itr = _btree_dict.find(idx, *_folded_compare); assert(itr.valid()); return itr.getKey(); } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index 2c2c39451c0..acb85c4e7f2 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -12,20 +12,20 @@ class IEnumStore; /** * Concrete dictionary for an enum store that extends the functionality of a unique store dictionary. */ -template <typename DictionaryT, typename UnorderedDictionaryT = vespalib::datastore::NoUnorderedDictionary> -class EnumStoreDictionary : public vespalib::datastore::UniqueStoreDictionary<DictionaryT, IEnumStoreDictionary, UnorderedDictionaryT> { +template <typename BTreeDictionaryT, typename HashDictionaryT = vespalib::datastore::NoHashDictionary> +class EnumStoreDictionary : public vespalib::datastore::UniqueStoreDictionary<BTreeDictionaryT, IEnumStoreDictionary, HashDictionaryT> { protected: using EntryRef = IEnumStoreDictionary::EntryRef; using Index = IEnumStoreDictionary::Index; - using DictionaryType = DictionaryT; + using BTreeDictionaryType = BTreeDictionaryT; private: using EnumVector = IEnumStoreDictionary::EnumVector; using IndexSet = IEnumStoreDictionary::IndexSet; using IndexVector = IEnumStoreDictionary::IndexVector; - using ParentUniqueStoreDictionary = vespalib::datastore::UniqueStoreDictionary<DictionaryT, IEnumStoreDictionary, UnorderedDictionaryT>; + using ParentUniqueStoreDictionary = vespalib::datastore::UniqueStoreDictionary<BTreeDictionaryT, IEnumStoreDictionary, HashDictionaryT>; using generation_t = IEnumStoreDictionary::generation_t; protected: - using ParentUniqueStoreDictionary::has_unordered_dictionary; + using ParentUniqueStoreDictionary::has_hash_dictionary; private: IEnumStore& _enumStore; @@ -37,9 +37,7 @@ public: ~EnumStoreDictionary() override; - const DictionaryT& get_raw_dictionary() const { return this->_dict; } - - void set_ref_counts(const EnumVector& hist) override; + const BTreeDictionaryT& get_raw_dictionary() const { return this->_btree_dict; } void free_unused_values(const vespalib::datastore::EntryComparator& cmp) override; @@ -58,8 +56,7 @@ public: Index remap_index(Index idx) override; void clear_all_posting_lists(std::function<void(EntryRef)> clearer) override; void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) override; - void sync_unordered_after_load() override; - EnumPostingTree& get_posting_dictionary() override; + bool normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) override; const EnumPostingTree& get_posting_dictionary() const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp index 0dd7ba426b1..c335c2064f1 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp @@ -2,6 +2,7 @@ #include "enum_store_loaders.h" #include "i_enum_store.h" +#include "i_enum_store_dictionary.h" #include <vespa/vespalib/util/array.hpp> namespace search::enumstore { @@ -19,6 +20,18 @@ EnumeratedLoaderBase::load_unique_values(const void* src, size_t available) assert(static_cast<size_t>(sz) == available); } +void +EnumeratedLoaderBase::release_enum_indexes() +{ + IndexVector().swap(_indexes); +} + +void +EnumeratedLoaderBase::free_unused_values() +{ + _store.free_unused_values(); +} + EnumeratedLoader::EnumeratedLoader(IEnumStore& store) : EnumeratedLoaderBase(store), _enums_histogram() @@ -28,15 +41,29 @@ EnumeratedLoader::EnumeratedLoader(IEnumStore& store) void EnumeratedLoader::set_ref_counts() { - _store.set_ref_counts(_enums_histogram); + assert(_enums_histogram.size() == _indexes.size()); + for (uint32_t i = 0; i < _indexes.size(); ++i) { + _store.set_ref_count(_indexes[i], _enums_histogram[i]); + } + EnumVector().swap(_enums_histogram); +} + +void +EnumeratedLoader::build_dictionary() +{ + _store.get_dictionary().build(_indexes); + release_enum_indexes(); } EnumeratedPostingsLoader::EnumeratedPostingsLoader(IEnumStore& store) : EnumeratedLoaderBase(store), - _loaded_enums() + _loaded_enums(), + _posting_indexes() { } +EnumeratedPostingsLoader::~EnumeratedPostingsLoader() = default; + bool EnumeratedPostingsLoader::is_folded_change(Index lhs, Index rhs) const { @@ -49,10 +76,20 @@ EnumeratedPostingsLoader::set_ref_count(Index idx, uint32_t ref_count) _store.set_ref_count(idx, ref_count); } +vespalib::ArrayRef<uint32_t> +EnumeratedPostingsLoader::initialize_empty_posting_indexes() +{ + vespalib::Array<uint32_t>(_indexes.size(), 0).swap(_posting_indexes); + return _posting_indexes; +} + void -EnumeratedPostingsLoader::free_unused_values() +EnumeratedPostingsLoader::build_dictionary() { - _store.free_unused_values(); + attribute::LoadedEnumAttributeVector().swap(_loaded_enums); + _store.get_dictionary().build_with_payload(_indexes, _posting_indexes); + release_enum_indexes(); + vespalib::Array<uint32_t>().swap(_posting_indexes); } } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h index eed97b50552..87705681dcf 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h @@ -17,13 +17,12 @@ protected: IEnumStore& _store; IndexVector _indexes; + void release_enum_indexes(); public: EnumeratedLoaderBase(IEnumStore& store); const IndexVector& get_enum_indexes() const { return _indexes; } void load_unique_values(const void* src, size_t available); - void release_enum_indexes() { - IndexVector().swap(_indexes); - } + void free_unused_values(); }; /** @@ -40,6 +39,7 @@ public: EnumVector(_indexes.size(), 0).swap(_enums_histogram); } void set_ref_counts(); + void build_dictionary(); }; /** @@ -48,9 +48,11 @@ public: class EnumeratedPostingsLoader : public EnumeratedLoaderBase { private: attribute::LoadedEnumAttributeVector _loaded_enums; + vespalib::Array<uint32_t> _posting_indexes; public: EnumeratedPostingsLoader(IEnumStore& store); + ~EnumeratedPostingsLoader(); attribute::LoadedEnumAttributeVector& get_loaded_enums() { return _loaded_enums; } void reserve_loaded_enums(size_t num_values) { _loaded_enums.reserve(num_values); @@ -60,7 +62,8 @@ public: } bool is_folded_change(Index lhs, Index rhs) const; void set_ref_count(Index idx, uint32_t ref_count); - void free_unused_values(); + vespalib::ArrayRef<uint32_t> initialize_empty_posting_indexes(); + void build_dictionary(); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index dc501dc9d89..fd523e227b0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -13,7 +13,7 @@ EnumAttribute<B>:: EnumAttribute(const vespalib::string &baseFileName, const AttributeVector::Config &cfg) : B(baseFileName, cfg), - _enumStore(cfg.fastSearch(), cfg.get_dictionary_config().getOrdering()) + _enumStore(cfg.fastSearch(), cfg.get_dictionary_config().getType()) { this->setEnum(true); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index 0961f7c87f1..5beafea0046 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -42,14 +42,15 @@ EnumStoreT<const char*>::load_unique_value(const void* src, } std::unique_ptr<vespalib::datastore::IUniqueStoreDictionary> -make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Ordering ordering, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare) +make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Type type, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare) { if (has_postings) { if (folded_compare) { return std::make_unique<EnumStoreFoldedDictionary>(store, std::move(compare), std::move(folded_compare)); } else { - switch (ordering) { - case search::DictionaryConfig::Ordering::UNORDERED: + switch (type) { + case search::DictionaryConfig::Type::HASH: + case search::DictionaryConfig::Type::BTREE_AND_HASH: return std::make_unique<EnumStoreDictionary<EnumPostingTree, vespalib::datastore::SimpleHashMap>>(store, std::move(compare)); default: return std::make_unique<EnumStoreDictionary<EnumPostingTree>>(store, std::move(compare)); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 8bff68b8408..6d77295db08 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -74,8 +74,8 @@ private: ssize_t load_unique_value(const void* src, size_t available, Index& idx); public: - EnumStoreT(bool has_postings, search::DictionaryConfig::Ordering ordering); - virtual ~EnumStoreT(); + EnumStoreT(bool has_postings, search::DictionaryConfig::Type type); + ~EnumStoreT() override; uint32_t get_ref_count(Index idx) const { return get_entry_base(idx).get_ref_count(); } void inc_ref_count(Index idx) { return get_entry_base(idx).inc_ref_count(); } @@ -97,13 +97,10 @@ public: ssize_t load_unique_values(const void* src, size_t available, IndexVector& idx) override; - void set_ref_counts(const EnumVector& hist) override { _dict->set_ref_counts(hist); } void freeze_dictionary() { _store.freeze(); } IEnumStoreDictionary& get_dictionary() override { return *_dict; } const IEnumStoreDictionary& get_dictionary() const override { return *_dict; } - EnumPostingTree& get_posting_dictionary() { return _dict->get_posting_dictionary(); } - const EnumPostingTree& get_posting_dictionary() const { return _dict->get_posting_dictionary(); } bool get_value(Index idx, EntryType& value) const; EntryType get_value(uint32_t idx) const { return get_value(Index(EntryRef(idx))); } @@ -214,7 +211,9 @@ public: }; std::unique_ptr<vespalib::datastore::IUniqueStoreDictionary> -make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Ordering ordering, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare); +make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Type type, + std::unique_ptr<vespalib::datastore::EntryComparator> compare, + std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare); template <> diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 69c412324e1..357026ab944 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -72,13 +72,13 @@ EnumStoreT<EntryT>::load_unique_value(const void* src, size_t available, Index& } template <typename EntryT> -EnumStoreT<EntryT>::EnumStoreT(bool has_postings, search::DictionaryConfig::Ordering ordering) +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, search::DictionaryConfig::Type type) : _store(), _dict(), _cached_values_memory_usage(), _cached_values_address_space_usage(0, 0, (1ull << 32)) { - _store.set_dictionary(make_enum_store_dictionary(*this, has_postings, ordering, + _store.set_dictionary(make_enum_store_dictionary(*this, has_postings, type, std::make_unique<ComparatorType>(_store.get_data_store()), (has_string_type() ? std::make_unique<FoldedComparatorType>(_store.get_data_store()) : @@ -116,9 +116,6 @@ ssize_t EnumStoreT<EntryT>::load_unique_values(const void* src, size_t available, IndexVector& idx) { ssize_t sz = load_unique_values_internal(src, available, idx); - if (sz >= 0) { - _dict->build(idx); - } return sz; } diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 9819ae81c15..321459078b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -55,7 +55,6 @@ public: virtual void write_value(BufferWriter& writer, Index idx) const = 0; virtual ssize_t load_unique_values(const void* src, size_t available, IndexVector& idx) = 0; virtual void set_ref_count(Index idx, uint32_t ref_count) = 0; - virtual void set_ref_counts(const EnumVector& histogram) = 0; virtual void free_value_if_unused(Index idx, IndexSet& unused) = 0; virtual void free_unused_values() = 0; virtual bool is_folded_change(Index idx1, Index idx2) const = 0; @@ -80,22 +79,6 @@ public: } virtual std::unique_ptr<Enumerator> make_enumerator() const = 0; - - template <typename TreeT> - void set_ref_counts(const EnumVector& hist, TreeT& tree) { - if (hist.empty()) { - return; - } - typename TreeT::Iterator ti(tree.begin()); - typedef EnumVector::const_iterator HistIT; - - for (HistIT hi(hist.begin()), hie(hist.end()); hi != hie; ++hi, ++ti) { - assert(ti.valid()); - set_ref_count(ti.getKey(), *hi); - } - assert(!ti.valid()); - free_unused_values(); - } }; } diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index 309f037ac84..f816177b06c 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -38,7 +38,6 @@ public: public: virtual ~IEnumStoreDictionary() = default; - virtual void set_ref_counts(const EnumVector& hist) = 0; virtual void free_unused_values(const vespalib::datastore::EntryComparator& cmp) = 0; virtual void free_unused_values(const IndexSet& to_remove, const vespalib::datastore::EntryComparator& cmp) = 0; @@ -53,8 +52,7 @@ public: virtual Index remap_index(Index idx) = 0; virtual void clear_all_posting_lists(std::function<void(EntryRef)> clearer) = 0; virtual void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) = 0; - virtual void sync_unordered_after_load() = 0; - virtual EnumPostingTree& get_posting_dictionary() = 0; + virtual bool normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) = 0; virtual const EnumPostingTree& get_posting_dictionary() const = 0; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index 71ee1640b32..bd3a8adb56e 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -100,7 +100,6 @@ MultiValueEnumAttribute<B, M>::load_enumerated_data(ReaderBase& attrReader, uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, vespalib::ConstArrayRef<EnumIndex>(loader.get_enum_indexes()), attribute::SaveLoadedEnum(loader.get_loaded_enums())); - loader.release_enum_indexes(); loader.sort_loaded_enums(); this->checkSetMaxValueCount(maxvc); } @@ -114,8 +113,9 @@ MultiValueEnumAttribute<B, M>::load_enumerated_data(ReaderBase& attrReader, uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, vespalib::ConstArrayRef<EnumIndex>(loader.get_enum_indexes()), attribute::SaveEnumHist(loader.get_enums_histogram())); - loader.release_enum_indexes(); loader.set_ref_counts(); + loader.build_dictionary(); + loader.free_unused_values(); this->checkSetMaxValueCount(maxvc); } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index c76571b151d..3cf51f43613 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -53,22 +53,18 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu uint32_t preve = 0; uint32_t refCount = 0; - auto& dict = _dictionary.get_posting_dictionary(); - auto itr = dict.begin(); - auto posting_itr = itr; - assert(itr.valid()); + vespalib::ConstArrayRef<EnumIndex> enum_indexes(loader.get_enum_indexes()); + assert(!enum_indexes.empty()); + auto posting_indexes = loader.initialize_empty_posting_indexes(); + uint32_t posting_enum = preve; for (const auto& elem : loaded_enums) { if (preve != elem.getEnum()) { assert(preve < elem.getEnum()); - loader.set_ref_count(itr.getKey(), refCount); + assert(elem.getEnum() < enum_indexes.size()); + loader.set_ref_count(enum_indexes[preve], refCount); refCount = 0; - while (preve != elem.getEnum()) { - ++itr; - assert(itr.valid()); - ++preve; - } - assert(itr.valid()); - if (loader.is_folded_change(posting_itr.getKey(), itr.getKey())) { + preve = elem.getEnum(); + if (loader.is_folded_change(enum_indexes[posting_enum], enum_indexes[preve])) { postings.removeDups(); newIndex = EntryRef(); _postingList.apply(newIndex, @@ -78,11 +74,9 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu &postings._removals[0], &postings._removals[0] + postings._removals.size()); - posting_itr.writeData(newIndex.ref()); - while (posting_itr != itr) { - ++posting_itr; - } + posting_indexes[posting_enum] = newIndex.ref(); postings.clear(); + posting_enum = elem.getEnum(); } } assert(refCount < std::numeric_limits<uint32_t>::max()); @@ -92,7 +86,7 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu postings.add(elem.getDocId(), elem.getWeight()); } assert(refCount != 0); - loader.set_ref_count(itr.getKey(), refCount); + loader.set_ref_count(enum_indexes[preve], refCount); postings.removeDups(); newIndex = EntryRef(); _postingList.apply(newIndex, @@ -100,9 +94,9 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu &postings._additions[0] + postings._additions.size(), &postings._removals[0], &postings._removals[0] + postings._removals.size()); - posting_itr.writeData(newIndex.ref()); + posting_indexes[posting_enum] = newIndex.ref(); + loader.build_dictionary(); loader.free_unused_values(); - _dictionary.sync_unordered_after_load(); } template <typename P> diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 10a998da46b..08db7b2b6b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -45,10 +45,9 @@ PostingListSearchContext::~PostingListSearchContext() = default; void PostingListSearchContext::lookupTerm(const vespalib::datastore::EntryComparator &comp) { - _lowerDictItr.lower_bound(_frozenDictionary.getRoot(), EnumIndex(), comp); - _upperDictItr = _lowerDictItr; - if (_upperDictItr.valid() && !comp.less(EnumIndex(), _upperDictItr.getKey())) { - ++_upperDictItr; + auto lookup_result = _dictionary.find_posting_list(comp, _frozenDictionary.getRoot()); + if (lookup_result.first.valid()) { + _pidx = lookup_result.second; _uniqueValues = 1u; } } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index f99ca61cba0..fb2617064e4 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -138,8 +138,13 @@ PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, co { if (!_merger.merge_done()) { _merger.reserveArray(128, wanted_hits); - diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, - max_per_group, cutoff_groups, cutoff_strict, _merger.getWritableArray(), _merger.getWritableStartPos()); + if (_uniqueValues == 1u && !_lowerDictItr.valid() && _pidx.valid()) { + diversity::diversify_single(_pidx, _postingList, wanted_hits, diversity_attr, + max_per_group, cutoff_groups, cutoff_strict, _merger.getWritableArray(), _merger.getWritableStartPos()); + } else { + diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, + max_per_group, cutoff_groups, cutoff_strict, _merger.getWritableArray(), _merger.getWritableStartPos()); + } _merger.merge(); } } diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 9fd4597c3d2..fee2520b132 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -124,45 +124,46 @@ PostingStore<DataT>::removeSparseBitVectors() } } if (needscan) { - typedef EnumPostingTree::Iterator EnumIterator; - auto& dict = _dictionary.get_posting_dictionary(); - for (EnumIterator dictItr = dict.begin(); dictItr.valid(); ++dictItr) { - if (!isBitVector(getTypeId(EntryRef(dictItr.getData())))) - continue; - EntryRef ref(dictItr.getData()); - RefType iRef(ref); - uint32_t typeId = getTypeId(iRef); - assert(isBitVector(typeId)); - assert(_bvs.find(ref.ref() )!= _bvs.end()); - BitVectorEntry *bve = getWBitVectorEntry(iRef); - BitVector &bv = *bve->_bv.get(); - uint32_t docFreq = bv.countTrueBits(); - if (bve->_tree.valid()) { - RefType iRef2(bve->_tree); - assert(isBTree(iRef2)); - const BTreeType *tree = getTreeEntry(iRef2); - assert(tree->size(_allocator) == docFreq); - (void) tree; - } - if (docFreq < _minBvDocFreq) { - dropBitVector(ref); - if (ref.valid()) { - iRef = ref; - typeId = getTypeId(iRef); - if (isBTree(typeId)) { - BTreeType *tree = getWTreeEntry(iRef); - normalizeTree(ref, tree, false); - } - } - dict.thaw(dictItr); - dictItr.writeData(ref.ref()); - res = true; - } - } + res = _dictionary.normalize_posting_lists([this](EntryRef posting_idx) -> EntryRef + { return consider_remove_sparse_bitvector(posting_idx); }); } return res; } +template <typename DataT> +typename PostingStore<DataT>::EntryRef +PostingStore<DataT>::consider_remove_sparse_bitvector(EntryRef ref) +{ + if (!ref.valid() || !isBitVector(getTypeId(EntryRef(ref)))) { + return ref; + } + RefType iRef(ref); + uint32_t typeId = getTypeId(iRef); + assert(isBitVector(typeId)); + assert(_bvs.find(ref.ref() )!= _bvs.end()); + BitVectorEntry *bve = getWBitVectorEntry(iRef); + BitVector &bv = *bve->_bv.get(); + uint32_t docFreq = bv.countTrueBits(); + if (bve->_tree.valid()) { + RefType iRef2(bve->_tree); + assert(isBTree(iRef2)); + const BTreeType *tree = getTreeEntry(iRef2); + assert(tree->size(_allocator) == docFreq); + (void) tree; + } + if (docFreq < _minBvDocFreq) { + dropBitVector(ref); + if (ref.valid()) { + iRef = ref; + typeId = getTypeId(iRef); + if (isBTree(typeId)) { + BTreeType *tree = getWTreeEntry(iRef); + normalizeTree(ref, tree, false); + } + } + } + return ref; +} template <typename DataT> void diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 5e41421e814..5ee1465d933 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -101,6 +101,7 @@ public: ~PostingStore(); bool removeSparseBitVectors() override; + EntryRef consider_remove_sparse_bitvector(EntryRef ref); static bool isBitVector(uint32_t typeId) { return typeId == BUFFERTYPE_BITVECTOR; } static bool isBTree(uint32_t typeId) { return typeId == BUFFERTYPE_BTREE; } bool isBTree(RefType ref) const { return isBTree(getTypeId(ref)); } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index ab7a75c4ffc..4d91c60ef4e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -209,7 +209,6 @@ SingleValueEnumAttribute<B>::load_enumerated_data(ReaderBase& attrReader, attrReader, loader.get_enum_indexes(), attribute::SaveLoadedEnum(loader.get_loaded_enums())); - loader.release_enum_indexes(); loader.sort_loaded_enums(); } @@ -224,8 +223,9 @@ SingleValueEnumAttribute<B>::load_enumerated_data(ReaderBase& attrReader, attrReader, loader.get_enum_indexes(), attribute::SaveEnumHist(loader.get_enums_histogram())); - loader.release_enum_indexes(); loader.set_ref_counts(); + loader.build_dictionary(); + loader.free_unused_values(); } template <typename B> diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java index f8eec00a340..cf9825b26cf 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ServiceType; /** @@ -16,4 +18,13 @@ public class ConfigServerApplication extends ConfigServerLikeApplication { super("zone-config-servers", NodeType.config, ClusterSpec.Type.admin, ServiceType.CONFIG_SERVER); } + /** + * A config server application has a particularly simple ApplicationInstanceId. + * + * @see InfraApplication#getApplicationInstanceId(Zone) + */ + public ApplicationInstanceId getApplicationInstanceId() { + return new ApplicationInstanceId(getApplicationId().application().value()); + } + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index da00ebc41e0..1bdf1ff67d9 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -12,12 +12,14 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.health.StateV1HealthModel; +import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import com.yahoo.vespa.service.model.ModelGenerator; import com.yahoo.vespa.service.monitor.InfraApplicationApi; @@ -94,8 +96,8 @@ public abstract class InfraApplication implements InfraApplicationApi { return serviceType; } - public ApplicationInstanceId getApplicationInstanceId() { - return new ApplicationInstanceId(applicationId.application().value()); + public ApplicationInstanceId getApplicationInstanceId(Zone zone) { + return ApplicationInstanceGenerator.toApplicationInstanceId(applicationId, zone); } public TenantId getTenantId() { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java index 37ebef5505e..d8e695f1a7e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index d0ecad5f27a..60e22639e8b 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -164,7 +164,7 @@ public class ApplicationInstanceGenerator { return new ServiceInstance(configId, hostName, status); } - private static ApplicationInstanceId toApplicationInstanceId(ApplicationId applicationId, Zone zone) { + public static ApplicationInstanceId toApplicationInstanceId(ApplicationId applicationId, Zone zone) { if (applicationId.equals(configServerApplicationId)) { // Removing this historical discrepancy would break orchestration during rollout. // An alternative may be to use a feature flag and flip it between releases, diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 61c74a263cf..4c574609df5 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -122,14 +122,14 @@ struct DistributorTest : Test, DistributorTestUtil { } } - distributor_component().removeNodesFromDB(makeDocumentBucket(document::BucketId(16, 1)), removedNodes); + operation_context().remove_nodes_from_bucket_database(makeDocumentBucket(document::BucketId(16, 1)), removedNodes); uint32_t flags(DatabaseUpdate::CREATE_IF_NONEXISTING | (resetTrusted ? DatabaseUpdate::RESET_TRUSTED : 0)); - distributor_component().updateBucketDatabase(makeDocumentBucket(document::BucketId(16, 1)), - changedNodes, - flags); + operation_context().update_bucket_database(makeDocumentBucket(document::BucketId(16, 1)), + changedNodes, + flags); } std::string retVal = dumpBucket(document::BucketId(16, 1)); @@ -572,8 +572,8 @@ TEST_F(DistributorTest, no_db_resurrection_for_bucket_not_owned_in_pending_state std::vector<BucketCopy> copies; copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2)); - distributor_component().updateBucketDatabase(makeDocumentBucket(nonOwnedBucket), copies, - DatabaseUpdate::CREATE_IF_NONEXISTING); + operation_context().update_bucket_database(makeDocumentBucket(nonOwnedBucket), copies, + DatabaseUpdate::CREATE_IF_NONEXISTING); EXPECT_EQ("NONEXISTING", dumpBucket(nonOwnedBucket)); } @@ -585,8 +585,8 @@ TEST_F(DistributorTest, added_db_buckets_without_gc_timestamp_implicitly_get_cur std::vector<BucketCopy> copies; copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2)); - distributor_component().updateBucketDatabase(makeDocumentBucket(bucket), copies, - DatabaseUpdate::CREATE_IF_NONEXISTING); + operation_context().update_bucket_database(makeDocumentBucket(bucket), copies, + DatabaseUpdate::CREATE_IF_NONEXISTING); BucketDatabase::Entry e(getBucket(bucket)); EXPECT_EQ(101234, e->getLastGarbageCollectionTime()); } diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp index b465edf5d16..7929cc1c906 100644 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -4,9 +4,9 @@ #include <vespa/document/test/make_bucket_space.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/distributor.h> -#include <vespa/storage/distributor/distributor_stripe.h> #include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/storage/distributor/distributorcomponent.h> +#include <vespa/storage/distributor/distributor_stripe.h> +#include <vespa/storage/distributor/distributor_stripe_component.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/text/stringtokenizer.h> @@ -259,7 +259,7 @@ void DistributorTestUtil::addIdealNodes(const document::BucketId& id) { // TODO STRIPE roundabout way of getting state bundle..! - addIdealNodes(*distributor_component().getClusterStateBundle().getBaselineClusterState(), id); + addIdealNodes(*operation_context().cluster_state_bundle().getBaselineClusterState(), id); } void @@ -351,12 +351,17 @@ DistributorTestUtil::getExternalOperationHandler() { return _distributor->external_operation_handler(); } -storage::distributor::DistributorComponent& +storage::distributor::DistributorStripeComponent& DistributorTestUtil::distributor_component() { // TODO STRIPE tests use this to indirectly access bucket space repos/DBs! return _distributor->distributor_component(); } +storage::distributor::DistributorOperationContext& +DistributorTestUtil::operation_context() { + return _distributor->distributor_component(); +} + bool DistributorTestUtil::tick() { framework::ThreadWaitInfo res( diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h index f450f2545db..d3c0445d5b5 100644 --- a/storage/src/tests/distributor/distributortestutil.h +++ b/storage/src/tests/distributor/distributortestutil.h @@ -21,10 +21,11 @@ class BucketDBUpdater; class Distributor; class DistributorBucketSpace; class DistributorBucketSpaceRepo; -class DistributorComponent; +class DistributorOperationContext; class DistributorStripe; -class IdealStateManager; +class DistributorStripeComponent; class ExternalOperationHandler; +class IdealStateManager; class Operation; // TODO STRIPE rename to DistributorStripeTestUtil? @@ -114,7 +115,8 @@ public: BucketDBUpdater& getBucketDBUpdater(); IdealStateManager& getIdealStateManager(); ExternalOperationHandler& getExternalOperationHandler(); - storage::distributor::DistributorComponent& distributor_component(); + storage::distributor::DistributorStripeComponent& distributor_component(); + storage::distributor::DistributorOperationContext& operation_context(); Distributor& getDistributor() { return *_distributor; diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp index 1829808990a..6d8086696b8 100644 --- a/storage/src/tests/distributor/externaloperationhandlertest.cpp +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -99,19 +99,19 @@ TEST_F(ExternalOperationHandlerTest, bucket_split_mask) { getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "16"); EXPECT_EQ(document::BucketId(16, 0xffff), - distributor_component().getBucketId(document::DocumentId( + operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( vespalib::make_string("id:ns:test:n=%d::", 0xffff)) ).stripUnused()); EXPECT_EQ(document::BucketId(16, 0), - distributor_component().getBucketId(document::DocumentId( + operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( vespalib::make_string("id:ns:test:n=%d::", 0x10000)) ).stripUnused()); EXPECT_EQ(document::BucketId(16, 0xffff), - distributor_component().getBucketId(document::DocumentId( + operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( vespalib::make_string("id:ns:test:n=%d::", 0xffff)) ).stripUnused()); EXPECT_EQ(document::BucketId(16, 0x100), - distributor_component().getBucketId(document::DocumentId( + operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( vespalib::make_string("id:ns:test:n=%d::", 0x100)) ).stripUnused()); close(); @@ -120,11 +120,11 @@ TEST_F(ExternalOperationHandlerTest, bucket_split_mask) { getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "20"); createLinks(); EXPECT_EQ(document::BucketId(20, 0x11111), - distributor_component().getBucketId(document::DocumentId( + operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( vespalib::make_string("id:ns:test:n=%d::", 0x111111)) ).stripUnused()); EXPECT_EQ(document::BucketId(20, 0x22222), - distributor_component().getBucketId(document::DocumentId( + operation_context().make_split_bit_constrained_bucket_id(document::DocumentId( vespalib::make_string("id:ns:test:n=%d::", 0x222222)) ).stripUnused()); } diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp index 1123c354ef4..cb671bb07f5 100644 --- a/storage/src/tests/distributor/getoperationtest.cpp +++ b/storage/src/tests/distributor/getoperationtest.cpp @@ -46,7 +46,7 @@ struct GetOperationTest : Test, DistributorTestUtil { createLinks(); docId = document::DocumentId("id:ns:text/html::uri"); - bucketId = distributor_component().getBucketId(docId); + bucketId = operation_context().make_split_bit_constrained_bucket_id(docId); }; void TearDown() override { diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index fd23dd5d656..ce9aa0a6800 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -64,17 +64,17 @@ struct IdealStateManagerTest : Test, DistributorTestUtil { TEST_F(IdealStateManagerTest, sibling) { EXPECT_EQ(document::BucketId(1,1), - getIdealStateManager().getDistributorComponent() - .getSibling(document::BucketId(1, 0))); + getIdealStateManager().operation_context() + .get_sibling(document::BucketId(1, 0))); EXPECT_EQ(document::BucketId(1,0), - getIdealStateManager().getDistributorComponent() - .getSibling(document::BucketId(1, 1))); + getIdealStateManager().operation_context() + .get_sibling(document::BucketId(1, 1))); EXPECT_EQ(document::BucketId(2,3), - getIdealStateManager().getDistributorComponent() - .getSibling(document::BucketId(2, 1))); + getIdealStateManager().operation_context() + .get_sibling(document::BucketId(2, 1))); EXPECT_EQ(document::BucketId(2,1), - getIdealStateManager().getDistributorComponent() - .getSibling(document::BucketId(2, 3))); + getIdealStateManager().operation_context() + .get_sibling(document::BucketId(2, 3))); } TEST_F(IdealStateManagerTest, status_page) { diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index a19708d6a27..aea251e81de 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -116,7 +116,7 @@ OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedundancy) { lib::IdealNodeCalculatorImpl idealNodeCalc; - auto &bucketSpaceRepo(distributor_component().getBucketSpaceRepo()); + auto &bucketSpaceRepo(operation_context().bucket_space_repo()); auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution()); idealNodeCalc.setClusterState(distributorBucketSpace.getClusterState()); @@ -145,7 +145,7 @@ TEST_F(OperationTargetResolverTest, simple) { TEST_F(OperationTargetResolverTest, multiple_nodes) { setupDistributor(1, 2, "storage:2 distributor:1"); - auto &bucketSpaceRepo(distributor_component().getBucketSpaceRepo()); + auto &bucketSpaceRepo(operation_context().bucket_space_repo()); auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); for (int i = 0; i < 100; ++i) { addNodesToBucketDB(BucketId(16, i), "0=0,1=0"); diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp index c510e08ab2a..ffd07ad9d60 100644 --- a/storage/src/tests/distributor/putoperationtest.cpp +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -104,7 +104,7 @@ document::BucketId PutOperationTest::createAndSendSampleDocument(vespalib::duration timeout) { auto doc = std::make_shared<Document>(doc_type(), DocumentId("id:test:testdoctype1::")); - document::BucketId id = distributor_component().getBucketId(doc->getId()); + document::BucketId id = operation_context().make_split_bit_constrained_bucket_id(doc->getId()); addIdealNodes(id); auto msg = std::make_shared<api::PutCommand>(makeDocumentBucket(document::BucketId(0)), doc, 0); @@ -150,7 +150,7 @@ TEST_F(PutOperationTest, bucket_database_gets_special_entry_when_CreateBucket_se // Database updated before CreateBucket is sent ASSERT_EQ("BucketId(0x4000000000008f09) : " "node(idx=0,crc=0x1,docs=0/0,bytes=0/0,trusted=true,active=true,ready=false)", - dumpBucket(distributor_component().getBucketId(doc->getId()))); + dumpBucket(operation_context().make_split_bit_constrained_bucket_id(doc->getId()))); ASSERT_EQ("Create bucket => 0,Put => 0", _sender.getCommands(true)); } @@ -197,7 +197,7 @@ TEST_F(PutOperationTest, return_success_if_op_acked_on_all_replicas_even_if_buck "id:test:testdoctype1::, timestamp 100, size 45) => 1", _sender.getCommands(true, true)); - distributor_component().removeNodeFromDB(makeDocumentBucket(document::BucketId(16, 0x1dd4)), 0); + operation_context().remove_node_from_bucket_database(makeDocumentBucket(document::BucketId(16, 0x1dd4)), 0); // If we get an ACK from the backend nodes, the operation has been persisted OK. // Even if the bucket has been removed from the DB in the meantime (usually would @@ -249,7 +249,7 @@ TEST_F(PutOperationTest, multiple_copies) { "node(idx=3,crc=0x1,docs=2/4,bytes=3/5,trusted=true,active=false,ready=false), " "node(idx=2,crc=0x1,docs=2/4,bytes=3/5,trusted=true,active=false,ready=false), " "node(idx=1,crc=0x1,docs=2/4,bytes=3/5,trusted=true,active=false,ready=false)", - dumpBucket(distributor_component().getBucketId(doc->getId()))); + dumpBucket(operation_context().make_split_bit_constrained_bucket_id(doc->getId()))); } TEST_F(PutOperationTest, multiple_copies_early_return_primary_required) { @@ -477,7 +477,7 @@ parseBucketInfoString(const std::string& nodeList) { std::string PutOperationTest::getNodes(const std::string& infoString) { Document::SP doc(createDummyDocument("test", "uri")); - document::BucketId bid(distributor_component().getBucketId(doc->getId())); + document::BucketId bid(operation_context().make_split_bit_constrained_bucket_id(doc->getId())); BucketInfo entry = parseBucketInfoString(infoString); @@ -519,7 +519,7 @@ TEST_F(PutOperationTest, replica_not_resurrected_in_db_when_node_down_in_active_ setupDistributor(Redundancy(3), NodeCount(3), "distributor:1 storage:3"); Document::SP doc(createDummyDocument("test", "uri")); - document::BucketId bId = distributor_component().getBucketId(doc->getId()); + document::BucketId bId = operation_context().make_split_bit_constrained_bucket_id(doc->getId()); addNodesToBucketDB(bId, "0=1/2/3/t,1=1/2/3/t,2=1/2/3/t"); @@ -536,14 +536,14 @@ TEST_F(PutOperationTest, replica_not_resurrected_in_db_when_node_down_in_active_ ASSERT_EQ("BucketId(0x4000000000000593) : " "node(idx=0,crc=0x7,docs=8/8,bytes=9/9,trusted=true,active=false,ready=false)", - dumpBucket(distributor_component().getBucketId(doc->getId()))); + dumpBucket(operation_context().make_split_bit_constrained_bucket_id(doc->getId()))); } TEST_F(PutOperationTest, replica_not_resurrected_in_db_when_node_down_in_pending_state) { setupDistributor(Redundancy(3), NodeCount(4), "version:1 distributor:1 storage:3"); auto doc = createDummyDocument("test", "uri"); - auto bucket = distributor_component().getBucketId(doc->getId()); + auto bucket = operation_context().make_split_bit_constrained_bucket_id(doc->getId()); addNodesToBucketDB(bucket, "0=1/2/3/t,1=1/2/3/t,2=1/2/3/t"); sendPut(createPut(doc)); @@ -577,7 +577,7 @@ TEST_F(PutOperationTest, replica_not_resurrected_in_db_when_node_down_in_pending TEST_F(PutOperationTest, put_is_failed_with_busy_if_target_down_in_pending_state) { setupDistributor(Redundancy(3), NodeCount(4), "version:1 distributor:1 storage:3"); auto doc = createDummyDocument("test", "test"); - auto bucket = distributor_component().getBucketId(doc->getId()); + auto bucket = operation_context().make_split_bit_constrained_bucket_id(doc->getId()); addNodesToBucketDB(bucket, "0=1/2/3/t,1=1/2/3/t,2=1/2/3/t"); getBucketDBUpdater().onSetSystemState( std::make_shared<api::SetSystemStateCommand>( @@ -597,7 +597,7 @@ TEST_F(PutOperationTest, send_to_retired_nodes_if_no_up_nodes_available) { "distributor:1 storage:2 .0.s:r .1.s:r"); Document::SP doc(createDummyDocument("test", "uri")); document::BucketId bucket( - distributor_component().getBucketId(doc->getId())); + operation_context().make_split_bit_constrained_bucket_id(doc->getId())); addNodesToBucketDB(bucket, "0=1/2/3/t,1=1/2/3/t"); sendPut(createPut(doc)); diff --git a/storage/src/tests/distributor/removeoperationtest.cpp b/storage/src/tests/distributor/removeoperationtest.cpp index de76379e854..c4892f342e7 100644 --- a/storage/src/tests/distributor/removeoperationtest.cpp +++ b/storage/src/tests/distributor/removeoperationtest.cpp @@ -24,7 +24,7 @@ struct RemoveOperationTest : Test, DistributorTestUtil { createLinks(); docId = document::DocumentId("id:test:test::uri"); - bucketId = distributor_component().getBucketId(docId); + bucketId = operation_context().make_split_bit_constrained_bucket_id(docId); enableDistributorClusterState("distributor:1 storage:4"); }; diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index 2f4c386e1ed..9b49f1347cc 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -78,8 +78,8 @@ struct StateCheckersTest : Test, DistributorTestUtil { { std::ostringstream ost; - c.siblingBucket = getIdealStateManager().getDistributorComponent() - .getSibling(c.getBucketId()); + c.siblingBucket = getIdealStateManager().operation_context() + .get_sibling(c.getBucketId()); std::vector<BucketDatabase::Entry> entries; getBucketDatabase(c.getBucketSpace()).getAll(c.getBucketId(), entries); diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 58556832f2d..dae94e41b46 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -309,7 +309,7 @@ TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState, } update->setCreateIfNonExistent(options._createIfNonExistent); - document::BucketId id = distributor_component().getBucketId(update->getId()); + document::BucketId id = operation_context().make_split_bit_constrained_bucket_id(update->getId()); document::BucketId id2 = document::BucketId(id.getUsedBits() + 1, id.getRawId()); if (bucketState.length()) { diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index ea9f0d86ac4..6620cf58571 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -60,7 +60,7 @@ UpdateOperationTest::sendUpdate(const std::string& bucketState, bool create_if_m document::DocumentId("id:ns:" + _html_type->getName() + "::1")); update->setCreateIfNonExistent(create_if_missing); - _bId = distributor_component().getBucketId(update->getId()); + _bId = operation_context().make_split_bit_constrained_bucket_id(update->getId()); addNodesToBucketDB(_bId, bucketState); diff --git a/storage/src/vespa/storage/common/distributorcomponent.h b/storage/src/vespa/storage/common/distributorcomponent.h index 27680b8c6ce..8d5920ecc77 100644 --- a/storage/src/vespa/storage/common/distributorcomponent.h +++ b/storage/src/vespa/storage/common/distributorcomponent.h @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * \class storage::DistributorComponent + * \class storage::DistributorStripeComponent * \ingroup common * * \brief Component class including some service layer specific information. diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 2b5423c60e4..d82d14831f4 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -3,26 +3,26 @@ vespa_add_library(storage_distributor SOURCES activecopy.cpp blockingoperationstarter.cpp - bucketdbupdater.cpp bucket_db_prune_elision.cpp + bucket_space_distribution_context.cpp + bucketdbupdater.cpp bucketgctimecalculator.cpp bucketlistmerger.cpp - bucket_space_distribution_context.cpp clusterinformation.cpp crypto_uuid_generator.cpp + distributor.cpp distributor_bucket_space.cpp distributor_bucket_space_repo.cpp - distributor.cpp distributor_host_info_reporter.cpp distributor_status.cpp distributor_stripe.cpp - distributorcomponent.cpp + distributor_stripe_component.cpp distributormessagesender.cpp distributormetricsset.cpp externaloperationhandler.cpp + ideal_service_layer_nodes_bundle.cpp idealstatemanager.cpp idealstatemetricsset.cpp - ideal_service_layer_nodes_bundle.cpp messagetracker.cpp nodeinfo.cpp operation_routing_snapshot.cpp @@ -31,8 +31,8 @@ vespa_add_library(storage_distributor operationtargetresolver.cpp operationtargetresolverimpl.cpp ownership_transfer_safe_time_point_calculator.cpp - pendingclusterstate.cpp pending_bucket_space_db_transition.cpp + pendingclusterstate.cpp pendingmessagetracker.cpp persistence_operation_metric_set.cpp persistencemessagetracker.cpp diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index 2a76ed5a26a..6735ea1e533 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -24,7 +24,7 @@ using document::BucketSpace; namespace storage::distributor { -BucketDBUpdater::BucketDBUpdater(DistributorInterface& owner, +BucketDBUpdater::BucketDBUpdater(DistributorStripeInterface& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, DistributorMessageSender& sender, diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index d80d823a7d1..375a5cee4e7 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -2,19 +2,19 @@ #pragma once #include "bucketlistmerger.h" -#include "messageguard.h" -#include "distributorcomponent.h" +#include "distributor_stripe_component.h" #include "distributormessagesender.h" -#include "pendingclusterstate.h" +#include "messageguard.h" #include "operation_routing_snapshot.h" #include "outdated_nodes_map.h" +#include "pendingclusterstate.h" #include <vespa/document/bucket/bucket.h> -#include <vespa/storageapi/message/bucket.h> -#include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/storagelink.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/messageapi/messagehandler.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageframework/generic/status/statusreporter.h> -#include <vespa/storageapi/messageapi/messagehandler.h> +#include <vespa/vdslib/state/clusterstate.h> #include <atomic> #include <list> #include <mutex> @@ -26,7 +26,7 @@ class XmlAttribute; namespace storage::distributor { -class DistributorInterface; +class DistributorStripeInterface; class BucketSpaceDistributionContext; class BucketDBUpdater : public framework::StatusReporter, @@ -34,7 +34,7 @@ class BucketDBUpdater : public framework::StatusReporter, { public: using OutdatedNodesMap = dbtransition::OutdatedNodesMap; - BucketDBUpdater(DistributorInterface& owner, + BucketDBUpdater(DistributorStripeInterface& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, DistributorMessageSender& sender, @@ -81,7 +81,7 @@ public: private: class MergeReplyGuard { public: - MergeReplyGuard(DistributorInterface& distributor_interface, const std::shared_ptr<api::MergeBucketReply>& reply) noexcept + MergeReplyGuard(DistributorStripeInterface& distributor_interface, const std::shared_ptr<api::MergeBucketReply>& reply) noexcept : _distributor_interface(distributor_interface), _reply(reply) {} ~MergeReplyGuard(); @@ -90,7 +90,7 @@ private: // than send it down void resetReply() { _reply.reset(); } private: - DistributorInterface& _distributor_interface; + DistributorStripeInterface& _distributor_interface; std::shared_ptr<api::MergeBucketReply> _reply; }; @@ -239,10 +239,10 @@ private: mutable bool _cachedOwned; }; - DistributorComponent _distributorComponent; + DistributorStripeComponent _distributorComponent; const DistributorNodeContext& _node_ctx; DistributorOperationContext& _op_ctx; - DistributorInterface& _distributor_interface; + DistributorStripeInterface& _distributor_interface; std::deque<std::pair<framework::MilliSecTime, BucketRequest> > _delayedRequests; std::map<uint64_t, BucketRequest> _sentMessages; std::unique_ptr<PendingClusterState> _pendingClusterState; diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 665478e68a2..f886640d1f9 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -29,9 +29,9 @@ using namespace std::chrono_literals; namespace storage::distributor { /* TODO STRIPE - * - need a DistributorComponent per stripe + * - need a DistributorStripeComponent per stripe * - or better, remove entirely! - * - probably also DistributorInterface since it's used to send + * - probably also DistributorStripeInterface since it's used to send * - metrics aggregation * - host info aggregation..!! * - handled if Distributor getMinReplica etc delegates to stripes? @@ -46,22 +46,16 @@ Distributor::Distributor(DistributorComponentRegister& compReg, HostInfo& hostInfoReporterRegistrar, ChainedMessageSender* messageSender) : StorageLink("distributor"), - DistributorInterface(), framework::StatusReporter("distributor", "Distributor"), _metrics(std::make_shared<DistributorMetricSet>()), _messageSender(messageSender), _stripe(std::make_unique<DistributorStripe>(compReg, *_metrics, node_identity, threadPool, doneInitHandler, manageActiveBucketCopies, *this)), - // TODO STRIPE remove once DistributorComponent no longer references bucket space repos - _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>(node_identity.node_index())), - _readOnlyBucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>(node_identity.node_index())), - // TODO STRIPE slim down - _component(*this, *_bucketSpaceRepo, *_readOnlyBucketSpaceRepo, compReg, "distributor"), + _component(compReg, "distributor"), _distributorStatusDelegate(compReg, *this, *this), _threadPool(threadPool), _tickResult(framework::ThreadWaitInfo::NO_MORE_CRITICAL_WORK_KNOWN), _metricUpdateHook(*this), - _metricLock(), _hostInfoReporter(*this, *this) { _component.registerMetric(*_metrics); @@ -82,11 +76,6 @@ Distributor::isInRecoveryMode() const noexcept { return _stripe->isInRecoveryMode(); } -int -Distributor::getDistributorIndex() const { - return _component.getIndex(); -} - const PendingMessageTracker& Distributor::getPendingMessageTracker() const { return _stripe->getPendingMessageTracker(); @@ -117,7 +106,7 @@ Distributor::getReadyOnlyBucketSpaceRepo() const noexcept { return _stripe->getReadOnlyBucketSpaceRepo();; } -storage::distributor::DistributorComponent& +storage::distributor::DistributorStripeComponent& Distributor::distributor_component() noexcept { // TODO STRIPE We need to grab the stripe's component since tests like to access // these things uncomfortably directly. @@ -169,27 +158,6 @@ Distributor::db_memory_sample_interval() const noexcept { return _stripe->db_memory_sample_interval(); } -bool Distributor::initializing() const { - return _stripe->initializing(); -} - -const lib::ClusterState* -Distributor::pendingClusterStateOrNull(const document::BucketSpace& space) const { - return bucket_db_updater().pendingClusterStateOrNull(space); -} - -void -Distributor::sendCommand(const std::shared_ptr<api::StorageCommand>&) -{ - assert(false); // TODO STRIPE -} - -void -Distributor::sendReply(const std::shared_ptr<api::StorageReply>&) -{ - assert(false); // TODO STRIPE -} - void Distributor::setNodeStateUp() { @@ -217,22 +185,11 @@ Distributor::onOpen() } } -void Distributor::send_shutdown_abort_reply(const std::shared_ptr<api::StorageMessage>& msg) { - api::StorageReply::UP reply( - std::dynamic_pointer_cast<api::StorageCommand>(msg)->makeReply()); - reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "Distributor is shutting down")); - sendUp(std::shared_ptr<api::StorageMessage>(reply.release())); -} - void Distributor::onClose() { LOG(debug, "Distributor::onClose invoked"); _stripe->close(); } -void Distributor::send_up_without_tracking(const std::shared_ptr<api::StorageMessage>&) { - assert(false); -} - void Distributor::sendUp(const std::shared_ptr<api::StorageMessage>& msg) { @@ -259,19 +216,6 @@ Distributor::onDown(const std::shared_ptr<api::StorageMessage>& msg) return _stripe->onDown(msg); } -void -Distributor::handleCompletedMerge( - const std::shared_ptr<api::MergeBucketReply>&) -{ - assert(false); -} - -bool -Distributor::isMaintenanceReply(const api::StorageReply&) const -{ - assert(false); -} - bool Distributor::handleReply(const std::shared_ptr<api::StorageReply>& reply) { @@ -298,16 +242,6 @@ Distributor::enableClusterStateBundle(const lib::ClusterStateBundle& state) _stripe->enableClusterStateBundle(state); } -OperationRoutingSnapshot Distributor::read_snapshot_for_bucket(const document::Bucket&) const { - abort(); -} - -void -Distributor::notifyDistributionChangeEnabled() -{ - _stripe->notifyDistributionChangeEnabled(); -} - void Distributor::storageDistributionChanged() { @@ -316,43 +250,6 @@ Distributor::storageDistributionChanged() } void -Distributor::recheckBucketInfo(uint16_t nodeIdx, const document::Bucket &bucket) { - bucket_db_updater().recheckBucketInfo(nodeIdx, bucket); -} - -namespace { - -class SplitChecker : public PendingMessageTracker::Checker -{ -public: - bool found; - uint8_t maxPri; - - SplitChecker(uint8_t maxP) : found(false), maxPri(maxP) {}; - - bool check(uint32_t msgType, uint16_t node, uint8_t pri) override { - (void) node; - (void) pri; - if (msgType == api::MessageType::SPLITBUCKET_ID && pri <= maxPri) { - found = true; - return false; - } - - return true; - } -}; - -} - -void -Distributor::checkBucketForSplit(document::BucketSpace, - const BucketDatabase::Entry&, - uint8_t) -{ - assert(false); -} - -void Distributor::enableNextDistribution() { _stripe->enableNextDistribution(); @@ -366,24 +263,6 @@ Distributor::propagateDefaultDistribution( _stripe->propagateDefaultDistribution(std::move(distribution)); } -void -Distributor::propagateClusterStates() -{ - assert(false); -} - -void -Distributor::signalWorkWasDone() -{ - _tickResult = framework::ThreadWaitInfo::MORE_WORK_ENQUEUED; -} - -bool -Distributor::workWasDone() const noexcept -{ - return !_tickResult.waitWanted(); -} - std::unordered_map<uint16_t, uint32_t> Distributor::getMinReplica() const { @@ -410,10 +289,6 @@ Distributor::propagateInternalScanMetricsToExternal() _stripe->propagateInternalScanMetricsToExternal(); } -void Distributor::maybe_update_bucket_db_memory_usage_stats() { - assert(false); -} - void Distributor::scanAllBuckets() { @@ -447,12 +322,6 @@ Distributor::enableNextConfig() _stripe->enableNextConfig(); // TODO STRIPE avoid redundant call } -void -Distributor::handleStatusRequests() -{ - assert(false); -} - vespalib::string Distributor::getReportContentType(const framework::HttpUrlPath& path) const { diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index c758dbd75e2..bfffe126b44 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -5,17 +5,17 @@ #include "bucket_spaces_stats_provider.h" #include "bucketdbupdater.h" #include "distributor_host_info_reporter.h" -#include "distributorinterface.h" +#include "distributor_stripe_interface.h" #include "externaloperationhandler.h" #include "idealstatemanager.h" #include "min_replica_provider.h" #include "pendingmessagetracker.h" #include "statusreporterdelegate.h" #include <vespa/config/config.h> +#include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/messagesender.h> #include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> -#include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/maintenance/maintenancescheduler.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> @@ -41,14 +41,13 @@ class OwnershipTransferSafeTimePointCalculator; class SimpleMaintenanceScanner; class ThrottlingOperationStarter; -class Distributor : public StorageLink, - public DistributorInterface, - public StatusDelegator, - public framework::StatusReporter, - public framework::TickingThread, - public MinReplicaProvider, - public BucketSpacesStatsProvider, - public NonTrackingMessageSender +class Distributor final + : public StorageLink, + public StatusDelegator, + public framework::StatusReporter, + public framework::TickingThread, + public MinReplicaProvider, + public BucketSpacesStatsProvider { public: Distributor(DistributorComponentRegister&, @@ -61,7 +60,7 @@ public: ~Distributor() override; - const ClusterContext& cluster_context() const override { + const ClusterContext& cluster_context() const { return _component.cluster_context(); } void onOpen() override; @@ -69,40 +68,18 @@ public: bool onDown(const std::shared_ptr<api::StorageMessage>&) override; void sendUp(const std::shared_ptr<api::StorageMessage>&) override; void sendDown(const std::shared_ptr<api::StorageMessage>&) override; - // Bypasses message tracker component. Thread safe. - void send_up_without_tracking(const std::shared_ptr<api::StorageMessage>&) override; - ChainedMessageSender& getMessageSender() override { - abort(); // TODO STRIPE - } - - DistributorMetricSet& getMetrics() override { return *_metrics; } - - const OperationSequencer& operation_sequencer() const noexcept override { - abort(); // TODO STRIPE - } - - const lib::ClusterState* pendingClusterStateOrNull(const document::BucketSpace&) const override; + DistributorMetricSet& getMetrics() { return *_metrics; } /** * Enables a new cluster state. Called after the bucket db updater has * retrieved all bucket info related to the change. */ - void enableClusterStateBundle(const lib::ClusterStateBundle& clusterStateBundle) override; - - /** - * Invoked when a pending cluster state for a distribution (config) - * change has been enabled. An invocation of storageDistributionChanged - * will eventually cause this method to be called, assuming the pending - * cluster state completed successfully. - */ - void notifyDistributionChangeEnabled() override; + void enableClusterStateBundle(const lib::ClusterStateBundle& clusterStateBundle); void storageDistributionChanged() override; - void recheckBucketInfo(uint16_t nodeIdx, const document::Bucket &bucket) override; - - bool handleReply(const std::shared_ptr<api::StorageReply>& reply) override; + bool handleReply(const std::shared_ptr<api::StorageReply>& reply); // StatusReporter implementation vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; @@ -115,57 +92,20 @@ public: virtual framework::ThreadWaitInfo doCriticalTick(framework::ThreadIndex) override; virtual framework::ThreadWaitInfo doNonCriticalTick(framework::ThreadIndex) override; - /** - * Checks whether a bucket needs to be split, and sends a split - * if so. - */ - void checkBucketForSplit(document::BucketSpace bucketSpace, - const BucketDatabase::Entry& e, - uint8_t priority) override; - - const lib::ClusterStateBundle& getClusterStateBundle() const override; - - /** - * @return Returns the states in which the distributors consider - * storage nodes to be up. - */ - const char* getStorageNodeUpStates() const override { - return "uri"; - } - - /** - * Called by bucket db updater after a merge has finished, and all the - * request bucket info operations have been performed as well. Passes the - * merge back to the operation that created it. - */ - void handleCompletedMerge(const std::shared_ptr<api::MergeBucketReply>& reply) override; - - bool initializing() const override; - - const DistributorConfiguration& getConfig() const override; + const lib::ClusterStateBundle& getClusterStateBundle() const; + const DistributorConfiguration& getConfig() const; bool isInRecoveryMode() const noexcept; - int getDistributorIndex() const override; - PendingMessageTracker& getPendingMessageTracker() override; - const PendingMessageTracker& getPendingMessageTracker() const override; + PendingMessageTracker& getPendingMessageTracker(); + const PendingMessageTracker& getPendingMessageTracker() const; DistributorBucketSpaceRepo& getBucketSpaceRepo() noexcept; const DistributorBucketSpaceRepo& getBucketSpaceRepo() const noexcept; DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() noexcept; const DistributorBucketSpaceRepo& getReadyOnlyBucketSpaceRepo() const noexcept; - storage::distributor::DistributorComponent& distributor_component() noexcept; - - void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; - void sendReply(const std::shared_ptr<api::StorageReply>&) override; - - const BucketGcTimeCalculator::BucketIdHasher& - getBucketIdHasher() const override { - abort(); // TODO STRIPE - } - - OperationRoutingSnapshot read_snapshot_for_bucket(const document::Bucket&) const override; + storage::distributor::DistributorStripeComponent& distributor_component() noexcept; class MetricUpdateHook : public framework::MetricUpdateHook { @@ -193,12 +133,6 @@ private: void setNodeStateUp(); bool handleMessage(const std::shared_ptr<api::StorageMessage>& msg); - bool isMaintenanceReply(const api::StorageReply& reply) const; - - void handleStatusRequests(); - void send_shutdown_abort_reply(const std::shared_ptr<api::StorageMessage>&); - void handle_or_propagate_message(const std::shared_ptr<api::StorageMessage>& msg); - void startExternalOperations(); // Accessors used by tests BucketDBUpdater& bucket_db_updater(); @@ -223,39 +157,21 @@ private: * Takes metric lock. */ void propagateInternalScanMetricsToExternal(); - void maybe_update_bucket_db_memory_usage_stats(); void scanAllBuckets(); void enableNextConfig(); - void signalWorkWasDone(); - bool workWasDone() const noexcept; - void enableNextDistribution(); void propagateDefaultDistribution(std::shared_ptr<const lib::Distribution>); - void propagateClusterStates(); std::shared_ptr<DistributorMetricSet> _metrics; - ChainedMessageSender* _messageSender; + ChainedMessageSender* _messageSender; // TODO STRIPE multiple stripes...! This is for proof of concept of wiring. - std::unique_ptr<DistributorStripe> _stripe; - - std::unique_ptr<DistributorBucketSpaceRepo> _bucketSpaceRepo; - // Read-only bucket space repo with DBs that only contain buckets transiently - // during cluster state transitions. Bucket set does not overlap that of _bucketSpaceRepo - // and the DBs are empty during non-transition phases. - std::unique_ptr<DistributorBucketSpaceRepo> _readOnlyBucketSpaceRepo; - storage::distributor::DistributorComponent _component; - - StatusReporterDelegate _distributorStatusDelegate; - - framework::TickingThreadPool& _threadPool; - - mutable std::vector<std::shared_ptr<DistributorStatus>> _statusToDo; - mutable std::vector<std::shared_ptr<DistributorStatus>> _fetchedStatusRequests; - - framework::ThreadWaitInfo _tickResult; - MetricUpdateHook _metricUpdateHook; - mutable std::mutex _metricLock; - DistributorHostInfoReporter _hostInfoReporter; + std::unique_ptr<DistributorStripe> _stripe; + storage::DistributorComponent _component; + StatusReporterDelegate _distributorStatusDelegate; + framework::TickingThreadPool& _threadPool; + framework::ThreadWaitInfo _tickResult; + MetricUpdateHook _metricUpdateHook; + DistributorHostInfoReporter _hostInfoReporter; }; } diff --git a/storage/src/vespa/storage/distributor/distributor_operation_context.h b/storage/src/vespa/storage/distributor/distributor_operation_context.h index 0b47c71e2e1..d9277a33088 100644 --- a/storage/src/vespa/storage/distributor/distributor_operation_context.h +++ b/storage/src/vespa/storage/distributor/distributor_operation_context.h @@ -33,11 +33,15 @@ public: const std::vector<BucketCopy>& changed_nodes, uint32_t update_flags = 0) = 0; virtual void remove_node_from_bucket_database(const document::Bucket& bucket, uint16_t node_index) = 0; + virtual void remove_nodes_from_bucket_database(const document::Bucket& bucket, + const std::vector<uint16_t>& nodes) = 0; virtual const DistributorBucketSpaceRepo& bucket_space_repo() const noexcept= 0; virtual DistributorBucketSpaceRepo& bucket_space_repo() noexcept = 0; virtual const DistributorBucketSpaceRepo& read_only_bucket_space_repo() const noexcept = 0; virtual DistributorBucketSpaceRepo& read_only_bucket_space_repo() noexcept = 0; virtual document::BucketId make_split_bit_constrained_bucket_id(const document::DocumentId& docId) const = 0; + virtual void recheck_bucket_info(uint16_t node_index, const document::Bucket& bucket) = 0; + virtual document::BucketId get_sibling(const document::BucketId& bid) const = 0; virtual const DistributorConfiguration& distributor_config() const noexcept = 0; virtual void send_inline_split_if_bucket_too_large(document::BucketSpace bucket_space, diff --git a/storage/src/vespa/storage/distributor/distributor_status.h b/storage/src/vespa/storage/distributor/distributor_status.h index 6783789949b..df405075308 100644 --- a/storage/src/vespa/storage/distributor/distributor_status.h +++ b/storage/src/vespa/storage/distributor/distributor_status.h @@ -8,12 +8,12 @@ namespace storage::framework { class HttpUrlPath; -class StatusReporter; +struct StatusReporter; } namespace storage::distributor { -class DelegatedStatusRequest; +struct DelegatedStatusRequest; // TODO STRIPE description class DistributorStatus { diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.cpp b/storage/src/vespa/storage/distributor/distributor_stripe.cpp index 4671e5ec9ca..e41c7940a0d 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe.cpp @@ -28,9 +28,9 @@ using namespace std::chrono_literals; namespace storage::distributor { /* TODO STRIPE - * - need a DistributorComponent per stripe + * - need a DistributorStripeComponent per stripe * - or better, remove entirely! - * - probably also DistributorInterface since it's used to send + * - probably also DistributorStripeInterface since it's used to send * - metrics aggregation */ DistributorStripe::DistributorStripe(DistributorComponentRegister& compReg, @@ -41,7 +41,7 @@ DistributorStripe::DistributorStripe(DistributorComponentRegister& compReg, bool manageActiveBucketCopies, ChainedMessageSender& messageSender) : StorageLink("distributor"), - DistributorInterface(), + DistributorStripeInterface(), framework::StatusReporter("distributor", "Distributor"), _clusterStateBundle(lib::ClusterState()), _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>(node_identity.node_index())), @@ -73,7 +73,6 @@ DistributorStripe::DistributorStripe(DistributorComponentRegister& compReg, _recoveryTimeStarted(_component.getClock()), _tickResult(framework::ThreadWaitInfo::NO_MORE_CRITICAL_WORK_KNOWN), _bucketIdHasher(std::make_unique<BucketGcTimeCalculator::BucketIdIdentityHasher>()), - _metricUpdateHook(*this), _metricLock(), _maintenanceStats(), _bucketSpacesStats(), diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.h b/storage/src/vespa/storage/distributor/distributor_stripe.h index dbf899a6de2..10b3f54d834 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe.h @@ -5,7 +5,7 @@ #include "bucket_spaces_stats_provider.h" #include "bucketdbupdater.h" #include "distributor_host_info_reporter.h" -#include "distributorinterface.h" +#include "distributor_stripe_interface.h" #include "externaloperationhandler.h" #include "idealstatemanager.h" #include "min_replica_provider.h" @@ -15,7 +15,7 @@ #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/messagesender.h> #include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> -#include <vespa/storage/distributor/distributorcomponent.h> +#include <vespa/storage/distributor/distributor_stripe_component.h> #include <vespa/storage/distributor/maintenance/maintenancescheduler.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> @@ -40,9 +40,12 @@ class OwnershipTransferSafeTimePointCalculator; class SimpleMaintenanceScanner; class ThrottlingOperationStarter; +/** + * TODO STRIPE add class comment. + */ class DistributorStripe final : public StorageLink, // TODO decouple - public DistributorInterface, + public DistributorStripeInterface, public StatusDelegator, public framework::StatusReporter, public framework::TickingThread, @@ -185,22 +188,6 @@ public: OperationRoutingSnapshot read_snapshot_for_bucket(const document::Bucket&) const override; - class MetricUpdateHook : public framework::MetricUpdateHook - { - public: - MetricUpdateHook(DistributorStripe& self) - : _self(self) - { - } - - void updateMetrics(const MetricLockGuard &) override { - _self.propagateInternalScanMetricsToExternal(); - } - - private: - DistributorStripe& _self; - }; - std::chrono::steady_clock::duration db_memory_sample_interval() const noexcept { return _db_memory_sample_interval; } @@ -281,7 +268,7 @@ private: // during cluster state transitions. Bucket set does not overlap that of _bucketSpaceRepo // and the DBs are empty during non-transition phases. std::unique_ptr<DistributorBucketSpaceRepo> _readOnlyBucketSpaceRepo; - storage::distributor::DistributorComponent _component; + storage::distributor::DistributorStripeComponent _component; DistributorMetricSet& _metrics; OperationOwner _operationOwner; @@ -332,7 +319,6 @@ private: framework::ThreadWaitInfo _tickResult; BucketDBMetricUpdater _bucketDBMetricUpdater; std::unique_ptr<BucketGcTimeCalculator::BucketIdHasher> _bucketIdHasher; - MetricUpdateHook _metricUpdateHook; mutable std::mutex _metricLock; /** * Maintenance stats for last completed database scan iteration. diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp index e5fe3c6c43c..ee131adf8ff 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp @@ -1,5 +1,6 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "distributorcomponent.h" +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "distributor_stripe_component.h" #include "distributor_bucket_space_repo.h" #include "distributor_bucket_space.h" #include "pendingmessagetracker.h" @@ -14,8 +15,8 @@ using document::BucketSpace; namespace storage::distributor { -DistributorComponent::DistributorComponent( - DistributorInterface& distributor, +DistributorStripeComponent::DistributorStripeComponent( + DistributorStripeInterface& distributor, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, DistributorComponentRegister& compReg, @@ -27,84 +28,22 @@ DistributorComponent::DistributorComponent( { } -DistributorComponent::~DistributorComponent() = default; +DistributorStripeComponent::~DistributorStripeComponent() = default; void -DistributorComponent::sendDown(const api::StorageMessage::SP& msg) +DistributorStripeComponent::sendDown(const api::StorageMessage::SP& msg) { _distributor.getMessageSender().sendDown(msg); } void -DistributorComponent::sendUp(const api::StorageMessage::SP& msg) +DistributorStripeComponent::sendUp(const api::StorageMessage::SP& msg) { _distributor.getMessageSender().sendUp(msg); } -const lib::ClusterStateBundle& -DistributorComponent::getClusterStateBundle() const -{ - return _distributor.getClusterStateBundle(); -}; - -api::StorageMessageAddress -DistributorComponent::nodeAddress(uint16_t nodeIndex) const -{ - return api::StorageMessageAddress::create(cluster_name_ptr(), lib::NodeType::STORAGE, nodeIndex); -} - -bool -DistributorComponent::checkDistribution(api::StorageCommand &cmd, const document::Bucket &bucket) -{ - auto &bucket_space(_bucketSpaceRepo.get(bucket.getBucketSpace())); - BucketOwnership bo(bucket_space.check_ownership_in_pending_and_current_state(bucket.getBucketId())); - if (!bo.isOwned()) { - std::string systemStateStr = bo.getNonOwnedState().toString(); - LOG(debug, - "Got message with wrong distribution, bucket %s sending back state '%s'", - bucket.toString().c_str(), systemStateStr.c_str()); - - api::StorageReply::UP reply(cmd.makeReply()); - api::ReturnCode ret(api::ReturnCode::WRONG_DISTRIBUTION, systemStateStr); - reply->setResult(ret); - sendUp(std::shared_ptr<api::StorageMessage>(reply.release())); - return false; - } - return true; -} - -void -DistributorComponent::removeNodesFromDB(const document::Bucket &bucket, const std::vector<uint16_t>& nodes) -{ - auto &bucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); - BucketDatabase::Entry dbentry = bucketSpace.getBucketDatabase().get(bucket.getBucketId()); - - if (dbentry.valid()) { - for (uint32_t i = 0; i < nodes.size(); ++i) { - if (dbentry->removeNode(nodes[i])) { - LOG(debug, - "Removed node %d from bucket %s. %u copies remaining", - nodes[i], - bucket.toString().c_str(), - dbentry->getNodeCount()); - } - } - - if (dbentry->getNodeCount() != 0) { - bucketSpace.getBucketDatabase().update(dbentry); - } else { - LOG(debug, - "After update, bucket %s now has no copies. " - "Removing from database.", - bucket.toString().c_str()); - - bucketSpace.getBucketDatabase().remove(bucket.getBucketId()); - } - } -} - void -DistributorComponent::enumerateUnavailableNodes( +DistributorStripeComponent::enumerateUnavailableNodes( std::vector<uint16_t>& unavailableNodes, const lib::ClusterState& s, const document::Bucket& bucket, @@ -184,10 +123,10 @@ UpdateBucketDatabaseProcessor::process_entry(BucketDatabase::Entry &entry) const } void -DistributorComponent::updateBucketDatabase( - const document::Bucket &bucket, - const std::vector<BucketCopy>& changedNodes, - uint32_t updateFlags) +DistributorStripeComponent::update_bucket_database( + const document::Bucket& bucket, + const std::vector<BucketCopy>& changed_nodes, + uint32_t update_flags) { auto &bucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); assert(!(bucket.getBucketId() == document::BucketId())); @@ -206,7 +145,7 @@ DistributorComponent::updateBucketDatabase( // bucket database (i.e. copies on nodes that are actually unavailable). const auto& available_nodes = bucketSpace.get_available_nodes(); bool found_down_node = false; - for (const auto& copy : changedNodes) { + for (const auto& copy : changed_nodes) { if (copy.getNode() >= available_nodes.size() || !available_nodes[copy.getNode()]) { found_down_node = true; break; @@ -216,46 +155,78 @@ DistributorComponent::updateBucketDatabase( // bucket copy vector std::vector<BucketCopy> up_nodes; if (found_down_node) { - up_nodes.reserve(changedNodes.size()); - for (uint32_t i = 0; i < changedNodes.size(); ++i) { - const BucketCopy& copy(changedNodes[i]); + up_nodes.reserve(changed_nodes.size()); + for (uint32_t i = 0; i < changed_nodes.size(); ++i) { + const BucketCopy& copy(changed_nodes[i]); if (copy.getNode() < available_nodes.size() && available_nodes[copy.getNode()]) { up_nodes.emplace_back(copy); } } } - UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changedNodes, bucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nodes(), (updateFlags & DatabaseUpdate::RESET_TRUSTED) != 0); + UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changed_nodes, bucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nodes(), (update_flags & DatabaseUpdate::RESET_TRUSTED) != 0); - bucketSpace.getBucketDatabase().process_update(bucket.getBucketId(), processor, (updateFlags & DatabaseUpdate::CREATE_IF_NONEXISTING) != 0); + bucketSpace.getBucketDatabase().process_update(bucket.getBucketId(), processor, (update_flags & DatabaseUpdate::CREATE_IF_NONEXISTING) != 0); } +// Implements DistributorNodeContext +api::StorageMessageAddress +DistributorStripeComponent::node_address(uint16_t node_index) const noexcept +{ + return api::StorageMessageAddress::create(cluster_name_ptr(), lib::NodeType::STORAGE, node_index); +} + + +// Implements DistributorOperationContext void -DistributorComponent::recheckBucketInfo(uint16_t nodeIdx, const document::Bucket &bucket) +DistributorStripeComponent::remove_nodes_from_bucket_database(const document::Bucket& bucket, + const std::vector<uint16_t>& nodes) { - _distributor.recheckBucketInfo(nodeIdx, bucket); + auto &bucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); + BucketDatabase::Entry dbentry = bucketSpace.getBucketDatabase().get(bucket.getBucketId()); + + if (dbentry.valid()) { + for (uint32_t i = 0; i < nodes.size(); ++i) { + if (dbentry->removeNode(nodes[i])) { + LOG(debug, + "Removed node %d from bucket %s. %u copies remaining", + nodes[i], + bucket.toString().c_str(), + dbentry->getNodeCount()); + } + } + + if (dbentry->getNodeCount() != 0) { + bucketSpace.getBucketDatabase().update(dbentry); + } else { + LOG(debug, + "After update, bucket %s now has no copies. " + "Removing from database.", + bucket.toString().c_str()); + + bucketSpace.getBucketDatabase().remove(bucket.getBucketId()); + } + } } document::BucketId -DistributorComponent::getBucketId(const document::DocumentId& docId) const +DistributorStripeComponent::make_split_bit_constrained_bucket_id(const document::DocumentId& doc_id) const { - document::BucketId id(getBucketIdFactory().getBucketId(docId)); + document::BucketId id(getBucketIdFactory().getBucketId(doc_id)); id.setUsedBits(_distributor.getConfig().getMinimalBucketSplit()); return id.stripUnused(); } -bool -DistributorComponent::storageNodeIsUp(document::BucketSpace bucketSpace, uint32_t nodeIndex) const +void +DistributorStripeComponent::recheck_bucket_info(uint16_t node_index, const document::Bucket& bucket) { - const lib::NodeState& ns = getClusterStateBundle().getDerivedClusterState(bucketSpace)->getNodeState( - lib::Node(lib::NodeType::STORAGE, nodeIndex)); - - return ns.getState().oneOf(_distributor.getStorageNodeUpStates()); + _distributor.recheckBucketInfo(node_index, bucket); } document::BucketId -DistributorComponent::getSibling(const document::BucketId& bid) const { +DistributorStripeComponent::get_sibling(const document::BucketId& bid) const +{ document::BucketId zeroBucket; document::BucketId oneBucket; @@ -279,28 +250,34 @@ DistributorComponent::getSibling(const document::BucketId& bid) const { } return (zeroBucket == bid) ? oneBucket : zeroBucket; -}; - -BucketDatabase::Entry -DistributorComponent::createAppropriateBucket(const document::Bucket &bucket) -{ - auto &bucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); - return bucketSpace.getBucketDatabase().createAppropriateBucket( - _distributor.getConfig().getMinimalBucketSplit(), - bucket.getBucketId()); } bool -DistributorComponent::has_pending_message(uint16_t node_index, - const document::Bucket& bucket, - uint32_t message_type) const +DistributorStripeComponent::has_pending_message(uint16_t node_index, + const document::Bucket& bucket, + uint32_t message_type) const { const auto& sender = static_cast<const DistributorMessageSender&>(getDistributor()); return sender.getPendingMessageTracker().hasPendingMessage(node_index, bucket, message_type); } +const lib::ClusterStateBundle& +DistributorStripeComponent::cluster_state_bundle() const +{ + return _distributor.getClusterStateBundle(); +} + +bool +DistributorStripeComponent::storage_node_is_up(document::BucketSpace bucket_space, uint32_t node_index) const +{ + const lib::NodeState& ns = cluster_state_bundle().getDerivedClusterState(bucket_space)->getNodeState( + lib::Node(lib::NodeType::STORAGE, node_index)); + + return ns.getState().oneOf(_distributor.getStorageNodeUpStates()); +} + std::unique_ptr<document::select::Node> -DistributorComponent::parse_selection(const vespalib::string& selection) const +DistributorStripeComponent::parse_selection(const vespalib::string& selection) const { document::select::Parser parser(*getTypeRepo()->documentTypeRepo, getBucketIdFactory()); return parser.parse(selection); diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.h b/storage/src/vespa/storage/distributor/distributor_stripe_component.h index 6a3620cbcf7..cb69b963271 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_component.h @@ -1,9 +1,9 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once #include "distributor_node_context.h" #include "distributor_operation_context.h" -#include "distributorinterface.h" +#include "distributor_stripe_interface.h" #include "document_selection_parser.h" #include "operationowner.h" #include "statechecker.h" @@ -26,156 +26,106 @@ struct DatabaseUpdate { /** * Takes care of subscribing to document manager config and * making those values available to other subcomponents. + * TODO STRIPE update class comment. */ -class DistributorComponent : public storage::DistributorComponent, - public DistributorNodeContext, - public DistributorOperationContext, - public DocumentSelectionParser +class DistributorStripeComponent : public storage::DistributorComponent, + public DistributorNodeContext, + public DistributorOperationContext, + public DocumentSelectionParser { public: - DistributorComponent(DistributorInterface& distributor, - DistributorBucketSpaceRepo& bucketSpaceRepo, - DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, - DistributorComponentRegister& compReg, - const std::string& name); + DistributorStripeComponent(DistributorStripeInterface& distributor, + DistributorBucketSpaceRepo& bucketSpaceRepo, + DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, + DistributorComponentRegister& compReg, + const std::string& name); - ~DistributorComponent() override; + ~DistributorStripeComponent() override; - /** - * Returns a reference to the current cluster state bundle. Valid until the - * next time the distributor main thread processes its message queue. - */ - const lib::ClusterStateBundle& getClusterStateBundle() const; + void sendDown(const api::StorageMessage::SP&); + void sendUp(const api::StorageMessage::SP&); - /** - * Returns the slobrok address of the given storage node. - */ - api::StorageMessageAddress nodeAddress(uint16_t nodeIndex) const; + DistributorStripeInterface& getDistributor() { return _distributor; } + + const DistributorStripeInterface& getDistributor() const { + return _distributor; + } + + // Implements DistributorNodeContext + const framework::Clock& clock() const noexcept override { return getClock(); } + const vespalib::string * cluster_name_ptr() const noexcept override { return cluster_context().cluster_name_ptr(); } + const document::BucketIdFactory& bucket_id_factory() const noexcept override { return getBucketIdFactory(); } + uint16_t node_index() const noexcept override { return getIndex(); } /** - * Returns true if the given storage node is in an "up state". + * Returns the slobrok address of the given storage node. */ - bool storageNodeIsUp(document::BucketSpace bucketSpace, uint32_t nodeIndex) const; + api::StorageMessageAddress node_address(uint16_t node_index) const noexcept override; + + // Implements DistributorOperationContext + api::Timestamp generate_unique_timestamp() override { return getUniqueTimestamp(); } /** - * Verifies that the given command has been received at the - * correct distributor based on the current system state. + * Simple API for the common case of modifying a single node. */ - bool checkDistribution(api::StorageCommand& cmd, const document::Bucket &bucket); + void update_bucket_database(const document::Bucket& bucket, + const BucketCopy& changed_node, + uint32_t update_flags = 0) override { + update_bucket_database(bucket, + toVector<BucketCopy>(changed_node), + update_flags); + } /** - * Removes the given bucket copies from the bucket database. - * If the resulting bucket is empty afterwards, removes the entire - * bucket entry from the bucket database. + * Adds the given copies to the bucket database. */ - void removeNodesFromDB(const document::Bucket &bucket, - const std::vector<uint16_t>& nodes); + virtual void update_bucket_database(const document::Bucket& bucket, + const std::vector<BucketCopy>& changed_nodes, + uint32_t update_flags = 0) override; /** * Removes a copy from the given bucket from the bucket database. * If the resulting bucket is empty afterwards, removes the entire * bucket entry from the bucket database. */ - void removeNodeFromDB(const document::Bucket &bucket, uint16_t node) { - removeNodesFromDB(bucket, toVector<uint16_t>(node)); + void remove_node_from_bucket_database(const document::Bucket& bucket, uint16_t node_index) override { + remove_nodes_from_bucket_database(bucket, toVector<uint16_t>(node_index)); } /** - * Adds the given copies to the bucket database. + * Removes the given bucket copies from the bucket database. + * If the resulting bucket is empty afterwards, removes the entire + * bucket entry from the bucket database. */ - void updateBucketDatabase( - const document::Bucket &bucket, - const std::vector<BucketCopy>& changedNodes, - uint32_t updateFlags = 0); + void remove_nodes_from_bucket_database(const document::Bucket& bucket, + const std::vector<uint16_t>& nodes) override; - /** - * Simple API for the common case of modifying a single node. - */ - void updateBucketDatabase( - const document::Bucket &bucket, - const BucketCopy& changedNode, - uint32_t updateFlags = 0) - { - updateBucketDatabase(bucket, - toVector<BucketCopy>(changedNode), - updateFlags); + const DistributorBucketSpaceRepo& bucket_space_repo() const noexcept override { + return _bucketSpaceRepo; + } + DistributorBucketSpaceRepo& bucket_space_repo() noexcept override { + return _bucketSpaceRepo; + } + const DistributorBucketSpaceRepo& read_only_bucket_space_repo() const noexcept override { + return _readOnlyBucketSpaceRepo; } + DistributorBucketSpaceRepo& read_only_bucket_space_repo() noexcept override { + return _readOnlyBucketSpaceRepo; + } + document::BucketId make_split_bit_constrained_bucket_id(const document::DocumentId& doc_id) const override; /** * Fetch bucket info about the given bucket from the given node. * Used when we get BUCKET_NOT_FOUND. */ - void recheckBucketInfo(uint16_t nodeIdx, const document::Bucket &bucket); - - /** - * Returns the bucket id corresponding to the given document id. - */ - document::BucketId getBucketId(const document::DocumentId& docId) const; - - void sendDown(const api::StorageMessage::SP&); - void sendUp(const api::StorageMessage::SP&); - - DistributorInterface& getDistributor() { return _distributor; } - - const DistributorInterface& getDistributor() const { - return _distributor; - } - - DistributorBucketSpaceRepo &getBucketSpaceRepo() { return _bucketSpaceRepo; } - const DistributorBucketSpaceRepo &getBucketSpaceRepo() const { return _bucketSpaceRepo; } - - DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() { return _readOnlyBucketSpaceRepo; } - const DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() const { return _readOnlyBucketSpaceRepo; } + void recheck_bucket_info(uint16_t node_index, const document::Bucket& bucket) override; /** * Finds a bucket that has the same direct parent as the given bucket * (i.e. split one bit less), but different bit in the most used bit. */ - document::BucketId getSibling(const document::BucketId& bid) const; - - /** - * Create a bucket that is split correctly according to other buckets that - * are in the bucket database. - */ - BucketDatabase::Entry createAppropriateBucket(const document::Bucket &bucket); + document::BucketId get_sibling(const document::BucketId& bid) const override; - // Implements DistributorNodeContext - const framework::Clock& clock() const noexcept override { return getClock(); } - const vespalib::string * cluster_name_ptr() const noexcept override { return cluster_context().cluster_name_ptr(); } - const document::BucketIdFactory& bucket_id_factory() const noexcept override { return getBucketIdFactory(); } - uint16_t node_index() const noexcept override { return getIndex(); } - api::StorageMessageAddress node_address(uint16_t node_index) const noexcept override { return nodeAddress(node_index); } - - // Implements DistributorOperationContext - api::Timestamp generate_unique_timestamp() override { return getUniqueTimestamp(); } - void update_bucket_database(const document::Bucket& bucket, - const BucketCopy& changed_node, - uint32_t update_flags = 0) override { - updateBucketDatabase(bucket, changed_node, update_flags); - } - virtual void update_bucket_database(const document::Bucket& bucket, - const std::vector<BucketCopy>& changed_nodes, - uint32_t update_flags = 0) override { - updateBucketDatabase(bucket, changed_nodes, update_flags); - } - void remove_node_from_bucket_database(const document::Bucket& bucket, uint16_t node_index) override { - removeNodeFromDB(bucket, node_index); - } - const DistributorBucketSpaceRepo& bucket_space_repo() const noexcept override { - return getBucketSpaceRepo(); - } - DistributorBucketSpaceRepo& bucket_space_repo() noexcept override { - return getBucketSpaceRepo(); - } - const DistributorBucketSpaceRepo& read_only_bucket_space_repo() const noexcept override { - return getReadOnlyBucketSpaceRepo(); - } - DistributorBucketSpaceRepo& read_only_bucket_space_repo() noexcept override { - return getReadOnlyBucketSpaceRepo(); - } - document::BucketId make_split_bit_constrained_bucket_id(const document::DocumentId& docId) const override { - return getBucketId(docId); - } const DistributorConfiguration& distributor_config() const noexcept override { return getDistributor().getConfig(); } @@ -196,12 +146,18 @@ public: const lib::ClusterState* pending_cluster_state_or_null(const document::BucketSpace& bucket_space) const override { return getDistributor().pendingClusterStateOrNull(bucket_space); } - const lib::ClusterStateBundle& cluster_state_bundle() const override { - return getClusterStateBundle(); - } - bool storage_node_is_up(document::BucketSpace bucket_space, uint32_t node_index) const override { - return storageNodeIsUp(bucket_space, node_index); - } + + /** + * Returns a reference to the current cluster state bundle. Valid until the + * next time the distributor main thread processes its message queue. + */ + const lib::ClusterStateBundle& cluster_state_bundle() const override; + + /** + * Returns true if the given storage node is in an "up state". + */ + bool storage_node_is_up(document::BucketSpace bucket_space, uint32_t node_index) const override; + const char* storage_node_up_states() const override { return getDistributor().getStorageNodeUpStates(); } @@ -209,14 +165,13 @@ public: // Implements DocumentSelectionParser std::unique_ptr<document::select::Node> parse_selection(const vespalib::string& selection) const override; - private: void enumerateUnavailableNodes( std::vector<uint16_t>& unavailableNodes, const lib::ClusterState& s, const document::Bucket& bucket, const std::vector<BucketCopy>& candidates) const; - DistributorInterface& _distributor; + DistributorStripeInterface& _distributor; protected: diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributor_stripe_interface.h index bf4ff9c4c99..d83acfabffc 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_interface.h @@ -18,7 +18,10 @@ namespace storage::distributor { class DistributorMetricSet; class PendingMessageTracker; -class DistributorInterface : public DistributorMessageSender +/** + * TODO STRIPE add class comment. + */ +class DistributorStripeInterface : public DistributorMessageSender { public: virtual PendingMessageTracker& getPendingMessageTracker() = 0; diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 3f4a6674761..6205321ce84 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -3,7 +3,7 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/bucket/bucketidfactory.h> -#include <vespa/storage/distributor/distributorcomponent.h> +#include <vespa/storage/distributor/distributor_stripe_component.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <atomic> #include <chrono> diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index c92f6a3dfcf..84fef955feb 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -25,7 +25,7 @@ namespace storage { namespace distributor { IdealStateManager::IdealStateManager( - DistributorInterface& owner, + DistributorStripeInterface& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, DistributorComponentRegister& compReg, @@ -69,7 +69,7 @@ IdealStateManager::iAmUp() const { Node node(NodeType::DISTRIBUTOR, _distributorComponent.getIndex()); // Assume that derived cluster states agree on distributor node being up - const auto &state = *_distributorComponent.getClusterStateBundle().getBaselineClusterState(); + const auto &state = *operation_context().cluster_state_bundle().getBaselineClusterState(); const lib::State &nodeState = state.getNodeState(node).getState(); const lib::State &clusterState = state.getClusterState(); diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index 19f88334889..363d66d8174 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "distributorcomponent.h" +#include "distributor_stripe_component.h" #include "statechecker.h" #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> #include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> @@ -11,7 +11,7 @@ namespace storage::distributor { class IdealStateMetricSet; class IdealStateOperation; -class DistributorInterface; +class DistributorStripeInterface; class SplitBucketStateChecker; /** @@ -34,7 +34,7 @@ class IdealStateManager : public framework::HtmlStatusReporter, { public: - IdealStateManager(DistributorInterface& owner, + IdealStateManager(DistributorStripeInterface& owner, DistributorBucketSpaceRepo& bucketSpaceRepo, DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, DistributorComponentRegister& compReg, @@ -77,7 +77,9 @@ public: getBucketStatus(out); } - DistributorComponent& getDistributorComponent() { return _distributorComponent; } + const DistributorNodeContext& node_context() const { return _distributorComponent; } + DistributorOperationContext& operation_context() { return _distributorComponent; } + const DistributorOperationContext& operation_context() const { return _distributorComponent; } DistributorBucketSpaceRepo &getBucketSpaceRepo() { return _bucketSpaceRepo; } const DistributorBucketSpaceRepo &getBucketSpaceRepo() const { return _bucketSpaceRepo; } @@ -100,8 +102,8 @@ private: std::vector<StateChecker::SP> _stateCheckers; SplitBucketStateChecker* _splitBucketStateChecker; - DistributorComponent _distributorComponent; - DistributorBucketSpaceRepo &_bucketSpaceRepo; + DistributorStripeComponent _distributorComponent; + DistributorBucketSpaceRepo& _bucketSpaceRepo; mutable bool _has_logged_phantom_replica_warning; bool iAmUp() const; diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp index ec8e4539ad1..d0fdd539b72 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statbucketoperation.h" -#include <vespa/storage/distributor/distributorcomponent.h> +#include <vespa/storage/distributor/distributor_stripe_component.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/stat.h> #include <vespa/storage/distributor/distributor_bucket_space.h> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index 71f5693329a..c9e983d4284 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -27,7 +27,7 @@ void GarbageCollectionOperation::onStart(DistributorMessageSender& sender) { for (auto node : nodes) { auto command = std::make_shared<api::RemoveLocationCommand>( - _manager->getDistributorComponent().getDistributor().getConfig().getGarbageCollectionSelection(), + _manager->operation_context().distributor_config().getGarbageCollectionSelection(), getBucket()); command->setPriority(_priority); @@ -51,7 +51,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, uint16_t node = _tracker.handleReply(*rep); if (!rep->getResult().failed()) { - _replica_info.emplace_back(_manager->getDistributorComponent().getUniqueTimestamp(), + _replica_info.emplace_back(_manager->operation_context().generate_unique_timestamp(), node, rep->getBucketInfo()); _max_documents_removed = std::max(_max_documents_removed, rep->documents_removed()); } else { @@ -69,11 +69,11 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&, void GarbageCollectionOperation::merge_received_bucket_info_into_db() { // TODO avoid two separate DB ops for this. Current API currently does not make this elegant. - _manager->getDistributorComponent().updateBucketDatabase(getBucket(), _replica_info); + _manager->operation_context().update_bucket_database(getBucket(), _replica_info); BucketDatabase::Entry dbentry = _bucketSpace->getBucketDatabase().get(getBucketId()); if (dbentry.valid()) { dbentry->setLastGarbageCollectionTime( - _manager->getDistributorComponent().getClock().getTimeInSeconds().getTime()); + _manager->node_context().clock().getTimeInSeconds().getTime()); _bucketSpace->getBucketDatabase().update(dbentry); } } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp index 44f7b5d3b49..c2c43f86c42 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp @@ -110,7 +110,7 @@ JoinOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP& rep.getSourceBuckets()); for (uint32_t i = 0; i < sourceBuckets.size(); i++) { document::Bucket sourceBucket(msg->getBucket().getBucketSpace(), sourceBuckets[i]); - _manager->getDistributorComponent().removeNodeFromDB(sourceBucket, node); + _manager->operation_context().remove_node_from_bucket_database(sourceBucket, node); } // Add new buckets. @@ -118,9 +118,9 @@ JoinOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP& LOG(debug, "Invalid bucketinfo for bucket %s returned in join", getBucketId().toString().c_str()); } else { - _manager->getDistributorComponent().updateBucketDatabase( + _manager->operation_context().update_bucket_database( getBucket(), - BucketCopy(_manager->getDistributorComponent().getUniqueTimestamp(), + BucketCopy(_manager->operation_context().generate_unique_timestamp(), node, rep.getBucketInfo()), DatabaseUpdate::CREATE_IF_NONEXISTING); @@ -130,7 +130,7 @@ JoinOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP& } else if (rep.getResult().getResult() == api::ReturnCode::BUCKET_NOT_FOUND && _bucketSpace->getBucketDatabase().get(getBucketId())->getNode(node) != 0) { - _manager->getDistributorComponent().recheckBucketInfo(node, getBucket()); + _manager->operation_context().recheck_bucket_info(node, getBucket()); LOGBP(warning, "Join failed to find %s: %s", getBucketId().toString().c_str(), rep.getResult().toString().c_str()); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index dfbff93757a..afb806e903a 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -144,7 +144,7 @@ MergeOperation::onStart(DistributorMessageSender& sender) auto msg = std::make_shared<api::MergeBucketCommand>( getBucket(), _mnodes, - _manager->getDistributorComponent().getUniqueTimestamp(), + _manager->operation_context().generate_unique_timestamp(), clusterState.getVersion()); // Due to merge forwarding/chaining semantics, we must always send @@ -162,7 +162,7 @@ MergeOperation::onStart(DistributorMessageSender& sender) sender.sendToNode(lib::NodeType::STORAGE, _mnodes[0].index, msg); - _sentMessageTime = _manager->getDistributorComponent().getClock().getTimeInSeconds(); + _sentMessageTime = _manager->node_context().clock().getTimeInSeconds(); } else { LOGBP(debug, "Unable to merge bucket %s, since only one copy is available. System state %s", @@ -230,7 +230,7 @@ MergeOperation::deleteSourceOnlyNodes( if (!sourceOnlyNodes.empty()) { _removeOperation = std::make_unique<RemoveBucketOperation>( - _manager->getDistributorComponent().cluster_context(), + _manager->node_context(), BucketAndNodes(getBucket(), sourceOnlyNodes)); // Must not send removes to source only copies if something has caused // pending load to the copy after the merge was sent! diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp index 3bd2406cd85..6b06657d713 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp @@ -39,7 +39,7 @@ RemoveBucketOperation::onStartInternal(DistributorMessageSender& sender) _ok = true; if (!getNodes().empty()) { - _manager->getDistributorComponent().removeNodesFromDB(getBucket(), getNodes()); + _manager->operation_context().remove_nodes_from_bucket_database(getBucket(), getNodes()); for (uint32_t i = 0; i < msgs.size(); ++i) { _tracker.queueCommand(msgs[i].second, msgs[i].first); } @@ -81,9 +81,9 @@ RemoveBucketOperation::onReceiveInternal(const std::shared_ptr<api::StorageReply vespalib::string(rep->getResult().getMessage()).c_str(), rep->getBucketInfo().toString().c_str()); - _manager->getDistributorComponent().updateBucketDatabase( + _manager->operation_context().update_bucket_database( getBucket(), - BucketCopy(_manager->getDistributorComponent().getUniqueTimestamp(), + BucketCopy(_manager->operation_context().generate_unique_timestamp(), node, rep->getBucketInfo()), DatabaseUpdate::CREATE_IF_NONEXISTING); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp index f05feae6dab..d244521140a 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp @@ -98,7 +98,7 @@ SetBucketStateOperation::onReceive(DistributorMessageSender& sender, } entry->updateNode( - BucketCopy(_manager->getDistributorComponent().getUniqueTimestamp(), + BucketCopy(_manager->operation_context().generate_unique_timestamp(), node, bInfo).setTrusted(copy->trusted())); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index 2b3c80f9401..a75e954c118 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -94,14 +94,14 @@ SplitOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP ost << sinfo.first << ","; BucketCopy copy( - BucketCopy(_manager->getDistributorComponent().getUniqueTimestamp(), + BucketCopy(_manager->operation_context().generate_unique_timestamp(), node, sinfo.second)); // Must reset trusted since otherwise trustedness of inconsistent // copies would be arbitrarily determined by which copy managed // to finish its split first. - _manager->getDistributorComponent().updateBucketDatabase( + _manager->operation_context().update_bucket_database( document::Bucket(msg->getBucket().getBucketSpace(), sinfo.first), copy, (DatabaseUpdate::CREATE_IF_NONEXISTING | DatabaseUpdate::RESET_TRUSTED)); @@ -111,7 +111,7 @@ SplitOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP rep.getResult().getResult() == api::ReturnCode::BUCKET_NOT_FOUND && _bucketSpace->getBucketDatabase().get(rep.getBucketId())->getNode(node) != 0) { - _manager->getDistributorComponent().recheckBucketInfo(node, getBucket()); + _manager->operation_context().recheck_bucket_info(node, getBucket()); LOGBP(debug, "Split failed for %s: bucket not found. Storage and " "distributor bucket databases might be out of sync: %s", getBucketId().toString().c_str(), diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.h b/storage/src/vespa/storage/distributor/persistencemessagetracker.h index 1421b4c2038..87b2d1e8de0 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.h +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.h @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "distributor_stripe_component.h" #include "distributormetricsset.h" -#include "distributorcomponent.h" #include "messagetracker.h" #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/messageapi/bucketinfocommand.h> diff --git a/storage/src/vespa/storage/distributor/statechecker.cpp b/storage/src/vespa/storage/distributor/statechecker.cpp index 22605d56a74..bbbe283077f 100644 --- a/storage/src/vespa/storage/distributor/statechecker.cpp +++ b/storage/src/vespa/storage/distributor/statechecker.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "statechecker.h" -#include "distributorcomponent.h" #include "distributor_bucket_space.h" +#include "distributor_stripe_component.h" #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> @@ -61,12 +61,12 @@ StateChecker::Result::createStoredResult( return Result(std::unique_ptr<ResultImpl>(new StoredResultImpl(std::move(operation), MaintenancePriority(priority)))); } -StateChecker::Context::Context(const DistributorComponent& c, +StateChecker::Context::Context(const DistributorStripeComponent& c, const DistributorBucketSpace &distributorBucketSpace, NodeMaintenanceStatsTracker& statsTracker, const document::Bucket &bucket_) : bucket(bucket_), - siblingBucket(c.getSibling(bucket.getBucketId())), + siblingBucket(c.get_sibling(bucket.getBucketId())), systemState(distributorBucketSpace.getClusterState()), pending_cluster_state(c.getDistributor().pendingClusterStateOrNull(bucket_.getBucketSpace())), distributorConfig(c.getDistributor().getConfig()), diff --git a/storage/src/vespa/storage/distributor/statechecker.h b/storage/src/vespa/storage/distributor/statechecker.h index 734b44bd7d1..44c45e62ec8 100644 --- a/storage/src/vespa/storage/distributor/statechecker.h +++ b/storage/src/vespa/storage/distributor/statechecker.h @@ -17,7 +17,7 @@ namespace storage { class DistributorConfiguration; } namespace storage::distributor { -class DistributorComponent; +class DistributorStripeComponent; class DistributorBucketSpace; class NodeMaintenanceStatsTracker; @@ -44,7 +44,7 @@ public: */ struct Context { - Context(const DistributorComponent&, + Context(const DistributorStripeComponent&, const DistributorBucketSpace &distributorBucketSpace, NodeMaintenanceStatsTracker&, const document::Bucket &bucket_); @@ -76,7 +76,7 @@ public: std::vector<uint16_t> idealState; std::unordered_set<uint16_t> unorderedIdealState; - const DistributorComponent& component; + const DistributorStripeComponent& component; const BucketDatabase& db; NodeMaintenanceStatsTracker& stats; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index e6b74d9df4c..9cc21925f52 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.api.bindings; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.restapi.RestApi; import java.time.Instant; import java.util.Objects; @@ -13,7 +14,7 @@ import java.util.Set; * @author bjorncs */ @JsonIgnoreProperties(ignoreUnknown = true) -public class SignedIdentityDocumentEntity { +public class SignedIdentityDocumentEntity implements RestApi.JacksonResponseEntity { @JsonProperty("signature") public final String signature; @JsonProperty("signing-key-version") public final int signingKeyVersion; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index f69bdc2a91d..547ea524041 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.communication; -import ai.vespa.util.http.VespaHttpClientBuilder; +import ai.vespa.util.http.hc4.VespaHttpClientBuilder; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.security.SslContextBuilder; diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 5c0574aad8a..9e06bb152fd 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -133,6 +133,7 @@ vespa_define_module( src/tests/tutorial/simple src/tests/tutorial/threads src/tests/typify + src/tests/unwind_message src/tests/util/bfloat16 src/tests/util/file_area_freelist src/tests/util/generationhandler diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 2e1eea55fe1..25ed566f57a 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -13,7 +13,7 @@ #include <vespa/log/log.h> LOG_SETUP("unique_store_test"); -enum class Ordering { ORDERED, UNORDERED }; +enum class DictionaryType { BTREE, HASH, BTREE_AND_HASH }; using namespace vespalib::datastore; using vespalib::ArrayRef; @@ -27,9 +27,9 @@ struct TestBaseValues { static std::vector<ValueType> values; }; -template <typename UniqueStoreTypeAndOrder> +template <typename UniqueStoreTypeAndDictionaryType> struct TestBase : public ::testing::Test { - using UniqueStoreType = typename UniqueStoreTypeAndOrder::UniqueStoreType; + using UniqueStoreType = typename UniqueStoreTypeAndDictionaryType::UniqueStoreType; using EntryRefType = typename UniqueStoreType::RefType; using ValueType = typename UniqueStoreType::EntryType; using ValueConstRefType = typename UniqueStoreType::EntryConstRefType; @@ -142,22 +142,22 @@ struct TestBase : public ::testing::Test { } }; -template <typename UniqueStoreTypeAndOrder> -TestBase<UniqueStoreTypeAndOrder>::TestBase() +template <typename UniqueStoreTypeAndDictionaryType> +TestBase<UniqueStoreTypeAndDictionaryType>::TestBase() : store(), refStore(), generation(1) { - switch (UniqueStoreTypeAndOrder::ordering) { - case Ordering::ORDERED: + switch (UniqueStoreTypeAndDictionaryType::dictionary_type) { + case DictionaryType::BTREE: break; default: store.set_dictionary(std::make_unique<UniqueStoreDictionary<uniquestore::DefaultDictionary, IUniqueStoreDictionary, SimpleHashMap>>(std::make_unique<CompareType>(store.get_data_store()))); } } -template <typename UniqueStoreTypeAndOrder> -TestBase<UniqueStoreTypeAndOrder>::~TestBase() = default; +template <typename UniqueStoreTypeAndDictionaryType> +TestBase<UniqueStoreTypeAndDictionaryType>::~TestBase() = default; using NumberUniqueStore = UniqueStore<uint32_t>; using StringUniqueStore = UniqueStore<std::string>; @@ -177,61 +177,61 @@ std::vector<double> TestBaseValues<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, struct OrderedNumberUniqueStore { using UniqueStoreType = NumberUniqueStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; struct OrderedStringUniqueStore { using UniqueStoreType = StringUniqueStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; struct OrderedCStringUniqueStore { using UniqueStoreType = CStringUniqueStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; struct OrderedDoubleUniqueStore { using UniqueStoreType = DoubleUniqueStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; struct OrderedSmallOffsetNumberUniqueStore { using UniqueStoreType = SmallOffsetNumberUniqueStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; struct UnorderedNumberUniqueStore { using UniqueStoreType = NumberUniqueStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; struct UnorderedStringUniqueStore { using UniqueStoreType = StringUniqueStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; struct UnorderedCStringUniqueStore { using UniqueStoreType = CStringUniqueStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; struct UnorderedDoubleUniqueStore { using UniqueStoreType = DoubleUniqueStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; struct UnorderedSmallOffsetNumberUniqueStore { using UniqueStoreType = SmallOffsetNumberUniqueStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; using UniqueStoreTestTypes = ::testing::Types<OrderedNumberUniqueStore, OrderedStringUniqueStore, OrderedCStringUniqueStore, OrderedDoubleUniqueStore, UnorderedNumberUniqueStore, UnorderedStringUniqueStore, UnorderedCStringUniqueStore, UnorderedDoubleUniqueStore>; diff --git a/vespalib/src/tests/require/require_test.cpp b/vespalib/src/tests/require/require_test.cpp index d1060cfe474..a97e7a362cc 100644 --- a/vespalib/src/tests/require/require_test.cpp +++ b/vespalib/src/tests/require/require_test.cpp @@ -2,7 +2,6 @@ #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/gtest/gtest.h> -#include <stdexcept> //----------------------------------------------------------------------------- @@ -39,27 +38,31 @@ void fail_require_eq() { } TEST(RequireTest, require_can_fail) { - using E = std::invalid_argument; + using E = vespalib::RequireFailedException; EXPECT_THROW( - { - try { fail_require(); } - catch(const E &e) { - fprintf(stderr, "e.what() is >>>%s<<<\n", e.what()); - throw; - } - }, E); + { + try { fail_require(); } + catch(const E &e) { + fprintf(stderr, "e.getMessage() is >>>%s<<<\n", e.getMessage().c_str()); + fprintf(stderr, "e.getLocation() is >>>%s<<<\n", e.getLocation().c_str()); + fprintf(stderr, "e.what() is >>>%s<<<\n", e.what()); + throw; + } + }, E); } TEST(RequireTest, require_eq_can_fail) { - using E = std::invalid_argument; + using E = vespalib::RequireFailedException; EXPECT_THROW( - { - try { fail_require_eq(); } - catch(const E &e) { - fprintf(stderr, "e.what() is >>>%s<<<\n", e.what()); - throw; - } - }, E); + { + try { fail_require_eq(); } + catch(const E &e) { + fprintf(stderr, "e.getMessage() is >>>%s<<<\n", e.getMessage().c_str()); + fprintf(stderr, "e.getLocation() is >>>%s<<<\n", e.getLocation().c_str()); + fprintf(stderr, "e.what() is >>>%s<<<\n", e.what()); + throw; + } + }, E); } //----------------------------------------------------------------------------- diff --git a/vespalib/src/tests/unwind_message/CMakeLists.txt b/vespalib/src/tests/unwind_message/CMakeLists.txt new file mode 100644 index 00000000000..92295ada035 --- /dev/null +++ b/vespalib/src/tests/unwind_message/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_unwind_message_test_app TEST + SOURCES + unwind_message_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_unwind_message_test_app COMMAND vespalib_unwind_message_test_app) diff --git a/vespalib/src/tests/unwind_message/unwind_message_test.cpp b/vespalib/src/tests/unwind_message/unwind_message_test.cpp new file mode 100644 index 00000000000..84245189842 --- /dev/null +++ b/vespalib/src/tests/unwind_message/unwind_message_test.cpp @@ -0,0 +1,91 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/unwind_message.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <stdexcept> + +using vespalib::unwind_msg; +using vespalib::UnwindMessage; +using E = std::invalid_argument; + +//----------------------------------------------------------------------------- + +struct MyCheck { + ~MyCheck() { + EXPECT_EQ(std::uncaught_exceptions(), 2); + } +}; + +struct MyObj { + UnwindMessage msg1 = UnwindMessage("this SHOULD be printed (1/4)"); + UnwindMessage msg2 = UnwindMessage("this should NOT be printed (1)"); + UnwindMessage msg3 = UnwindMessage("this SHOULD be printed (2/4)"); + ~MyObj() { + EXPECT_EQ(std::uncaught_exceptions(), 1); + auto not_printed_1 = std::move(msg2); + try { + MyCheck my_check; + auto printed_1 = std::move(msg1); + throw E("next level"); + } catch (const E &) {} + } +}; + +TEST(UnwindMessageTest, unwind_messages_are_printed_when_appropriate) { + auto not_printed_5 = unwind_msg("this should NOT be printed (%d)", 5); + UNWIND_MSG("this should NOT be printed (%d)", 4); + EXPECT_THROW( + { + EXPECT_EQ(std::uncaught_exceptions(), 0); + auto printed_4 = unwind_msg("this SHOULD be printed (%d/%d)", 4, 4); + UNWIND_MSG("this SHOULD be printed (%d/%d)", 3, 4); + { + auto not_printed_3 = unwind_msg("this should NOT be printed (%d)", 3); + UNWIND_MSG("this should NOT be printed (%d)", 2); + } + MyObj my_obj; + throw E("just testing"); + }, E); +} + +//----------------------------------------------------------------------------- + +TEST(UnwindMessageTest, unwind_message_with_location) { + EXPECT_THROW( + { + UNWIND_MSG("%s message with location information", VESPA_STRLOC.c_str()); + throw E("just testing"); + }, E); +} + +//----------------------------------------------------------------------------- + +void my_bad_call() { + throw E("just testing"); +} + +TEST(UnwindMessageTest, unwind_message_from_UNWIND_DO_macro_calling_a_function) { + EXPECT_THROW( + { + UNWIND_DO(my_bad_call()); + }, E); +} + +//----------------------------------------------------------------------------- + +TEST(UnwindMessageTest, unwind_message_from_UNWIND_DO_macro_with_inline_code) { + EXPECT_THROW( + { + UNWIND_DO( + int a = 1; + int b = 2; + int c = a + b; + (void) c; + throw E("oops"); + ); + }, E); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index 2c495ca9a1e..d31139efc76 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -48,6 +48,7 @@ public: virtual void build(vespalib::ConstArrayRef<EntryRef> refs) = 0; virtual void build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, vespalib::ConstArrayRef<uint32_t> payloads) = 0; virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0; + virtual bool get_has_hash_dictionary() const = 0; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index c5630dbabfb..8fbff3f9d93 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -9,27 +9,40 @@ namespace vespalib::datastore { class EntryComparatorWrapper; -class NoUnorderedDictionary; +class NoHashDictionary; -template <typename UnorderedDictionaryT> -class UniqueStoreUnorderedDictionaryBase +template <typename BTreeDictionaryT> +class UniqueStoreBTreeDictionaryBase { protected: - UnorderedDictionaryT _unordered_dict; + BTreeDictionaryT _btree_dict; public: - static constexpr bool has_unordered_dictionary = true; - UniqueStoreUnorderedDictionaryBase(std::unique_ptr<EntryComparator> compare) - : _unordered_dict(std::move(compare)) + static constexpr bool has_btree_dictionary = true; + UniqueStoreBTreeDictionaryBase() + : _btree_dict() + { + } +}; + +template <typename HashDictionaryT> +class UniqueStoreHashDictionaryBase +{ +protected: + HashDictionaryT _hash_dict; +public: + static constexpr bool has_hash_dictionary = true; + UniqueStoreHashDictionaryBase(std::unique_ptr<EntryComparator> compare) + : _hash_dict(std::move(compare)) { } }; template <> -class UniqueStoreUnorderedDictionaryBase<NoUnorderedDictionary> +class UniqueStoreHashDictionaryBase<NoHashDictionary> { public: - static constexpr bool has_unordered_dictionary = false; - UniqueStoreUnorderedDictionaryBase(std::unique_ptr<EntryComparator>) + static constexpr bool has_hash_dictionary = false; + UniqueStoreHashDictionaryBase(std::unique_ptr<EntryComparator>) { } }; @@ -37,12 +50,12 @@ public: /** * A dictionary for unique store. Mostly accessed via base class. */ -template <typename DictionaryT, typename ParentT = IUniqueStoreDictionary, typename UnorderedDictionaryT = NoUnorderedDictionary> -class UniqueStoreDictionary : public ParentT, public UniqueStoreUnorderedDictionaryBase<UnorderedDictionaryT> { +template <typename BTreeDictionaryT, typename ParentT = IUniqueStoreDictionary, typename HashDictionaryT = NoHashDictionary> +class UniqueStoreDictionary : public ParentT, public UniqueStoreBTreeDictionaryBase<BTreeDictionaryT>, public UniqueStoreHashDictionaryBase<HashDictionaryT> { protected: - using DictionaryType = DictionaryT; - using DataType = typename DictionaryType::DataType; - using FrozenView = typename DictionaryType::FrozenView; + using BTreeDictionaryType = BTreeDictionaryT; + using DataType = typename BTreeDictionaryType::DataType; + using FrozenView = typename BTreeDictionaryType::FrozenView; using ReadSnapshot = typename ParentT::ReadSnapshot; using generation_t = typename ParentT::generation_t; @@ -57,11 +70,9 @@ protected: void foreach_key(std::function<void(EntryRef)> callback) const override; }; - DictionaryType _dict; - public: - using UniqueStoreUnorderedDictionaryBase<UnorderedDictionaryT>::has_unordered_dictionary; - static constexpr bool has_ordered_dictionary = true; + using UniqueStoreBTreeDictionaryBase<BTreeDictionaryT>::has_btree_dictionary; + using UniqueStoreHashDictionaryBase<HashDictionaryT>::has_hash_dictionary; UniqueStoreDictionary(std::unique_ptr<EntryComparator> compare); ~UniqueStoreDictionary() override; void freeze() override; @@ -77,6 +88,7 @@ public: void build(vespalib::ConstArrayRef<EntryRef> refs) override; void build_with_payload(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> payloads) override; std::unique_ptr<ReadSnapshot> get_read_snapshot() const override; + bool get_has_hash_dictionary() const override; }; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index 963a2dc72a1..ae8a85985fd 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -16,16 +16,16 @@ namespace vespalib::datastore { -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>:: +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: ReadSnapshotImpl::ReadSnapshotImpl(FrozenView frozen_view) : _frozen_view(frozen_view) { } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> size_t -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>:: +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: ReadSnapshotImpl::count(const EntryComparator& comp) const { auto itr = _frozen_view.lowerBound(EntryRef(), comp); @@ -35,9 +35,9 @@ ReadSnapshotImpl::count(const EntryComparator& comp) const return 0u; } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> size_t -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>:: +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: ReadSnapshotImpl::count_in_range(const EntryComparator& low, const EntryComparator& high) const { @@ -49,124 +49,124 @@ ReadSnapshotImpl::count_in_range(const EntryComparator& low, return high_itr - low_itr; } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>:: +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: ReadSnapshotImpl::foreach_key(std::function<void(EntryRef)> callback) const { _frozen_view.foreach_key(callback); } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::UniqueStoreDictionary(std::unique_ptr<EntryComparator> compare) +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::UniqueStoreDictionary(std::unique_ptr<EntryComparator> compare) : ParentT(), - UniqueStoreUnorderedDictionaryBase<UnorderedDictionaryT>(std::move(compare)), - _dict() + UniqueStoreBTreeDictionaryBase<BTreeDictionaryT>(), + UniqueStoreHashDictionaryBase<HashDictionaryT>(std::move(compare)) { } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::~UniqueStoreDictionary() = default; +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::~UniqueStoreDictionary() = default; -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::freeze() +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::freeze() { - _dict.getAllocator().freeze(); + this->_btree_dict.getAllocator().freeze(); } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::transfer_hold_lists(generation_t generation) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::transfer_hold_lists(generation_t generation) { - _dict.getAllocator().transferHoldLists(generation); - if constexpr (has_unordered_dictionary) { - this->_unordered_dict.transfer_hold_lists(generation); + this->_btree_dict.getAllocator().transferHoldLists(generation); + if constexpr (has_hash_dictionary) { + this->_hash_dict.transfer_hold_lists(generation); } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::trim_hold_lists(generation_t firstUsed) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::trim_hold_lists(generation_t firstUsed) { - _dict.getAllocator().trimHoldLists(firstUsed); - if constexpr (has_unordered_dictionary) { - this->_unordered_dict.trim_hold_lists(firstUsed); + this->_btree_dict.getAllocator().trimHoldLists(firstUsed); + if constexpr (has_hash_dictionary) { + this->_hash_dict.trim_hold_lists(firstUsed); } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> UniqueStoreAddResult -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::add(const EntryComparator &comp, +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::add(const EntryComparator &comp, std::function<EntryRef(void)> insertEntry) { - auto itr = _dict.lowerBound(EntryRef(), comp); + auto itr = this->_btree_dict.lowerBound(EntryRef(), comp); if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { - if constexpr (has_unordered_dictionary) { - auto* result = this->_unordered_dict.find(comp, EntryRef()); + if constexpr (has_hash_dictionary) { + auto* result = this->_hash_dict.find(comp, EntryRef()); assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); } return UniqueStoreAddResult(itr.getKey(), false); } else { EntryRef newRef = insertEntry(); - _dict.insert(itr, newRef, DataType()); - if constexpr (has_unordered_dictionary) { - std::function<EntryRef(void)> insert_unordered_entry([newRef]() noexcept -> EntryRef { return newRef; }); - auto& add_result = this->_unordered_dict.add(comp, newRef, insert_unordered_entry); + this->_btree_dict.insert(itr, newRef, DataType()); + if constexpr (has_hash_dictionary) { + std::function<EntryRef(void)> insert_hash_entry([newRef]() noexcept -> EntryRef { return newRef; }); + auto& add_result = this->_hash_dict.add(comp, newRef, insert_hash_entry); assert(add_result.first.load_relaxed() == newRef); } return UniqueStoreAddResult(newRef, true); } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> EntryRef -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::find(const EntryComparator &comp) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::find(const EntryComparator &comp) { - auto itr = _dict.lowerBound(EntryRef(), comp); + auto itr = this->_btree_dict.lowerBound(EntryRef(), comp); if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { - if constexpr (has_unordered_dictionary) { - auto* result = this->_unordered_dict.find(comp, EntryRef()); + if constexpr (has_hash_dictionary) { + auto* result = this->_hash_dict.find(comp, EntryRef()); assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); } return itr.getKey(); } else { - if constexpr (has_unordered_dictionary) { - auto* result = this->_unordered_dict.find(comp, EntryRef()); + if constexpr (has_hash_dictionary) { + auto* result = this->_hash_dict.find(comp, EntryRef()); assert(result == nullptr); } return EntryRef(); } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::remove(const EntryComparator &comp, EntryRef ref) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::remove(const EntryComparator &comp, EntryRef ref) { assert(ref.valid()); - auto itr = _dict.lowerBound(ref, comp); + auto itr = this->_btree_dict.lowerBound(ref, comp); assert(itr.valid() && itr.getKey() == ref); - _dict.remove(itr); - if constexpr (has_unordered_dictionary) { - auto *result = this->_unordered_dict.remove(comp, ref); + this->_btree_dict.remove(itr); + if constexpr (has_hash_dictionary) { + auto *result = this->_hash_dict.remove(comp, ref); assert(result != nullptr && result->first.load_relaxed() == ref); } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::move_entries(ICompactable &compactable) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_entries(ICompactable &compactable) { - auto itr = _dict.begin(); + auto itr = this->_btree_dict.begin(); while (itr.valid()) { EntryRef oldRef(itr.getKey()); EntryRef newRef(compactable.move(oldRef)); if (newRef != oldRef) { - _dict.thaw(itr); + this->_btree_dict.thaw(itr); itr.writeKey(newRef); - if constexpr (has_unordered_dictionary) { - auto result = this->_unordered_dict.find(this->_unordered_dict.get_default_comparator(), oldRef); + if constexpr (has_hash_dictionary) { + auto result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), oldRef); assert(result != nullptr && result->first.load_relaxed() == oldRef); result->first.store_release(newRef); } @@ -175,29 +175,29 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::move_entries( } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> uint32_t -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::get_num_uniques() const +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::get_num_uniques() const { - return _dict.size(); + return this->_btree_dict.size(); } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> vespalib::MemoryUsage -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::get_memory_usage() const +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::get_memory_usage() const { - return _dict.getMemoryUsage(); + return this->_btree_dict.getMemoryUsage(); } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build(vespalib::ConstArrayRef<EntryRef> refs, +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::build(vespalib::ConstArrayRef<EntryRef> refs, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) { assert(refs.size() == ref_counts.size()); assert(!refs.empty()); - typename DictionaryType::Builder builder(_dict.getAllocator()); + typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); for (size_t i = 1; i < refs.size(); ++i) { if (ref_counts[i] != 0u) { builder.insert(refs[i], DataType()); @@ -205,44 +205,44 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build(vespali hold(refs[i]); } } - _dict.assign(builder); - if constexpr (has_unordered_dictionary) { + this->_btree_dict.assign(builder); + if constexpr (has_hash_dictionary) { for (size_t i = 1; i < refs.size(); ++i) { if (ref_counts[i] != 0u) { EntryRef ref = refs[i]; - std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); - auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); + std::function<EntryRef(void)> insert_hash_entry([ref]() noexcept -> EntryRef { return ref; }); + auto& add_result = this->_hash_dict.add(this->_hash_dict.get_default_comparator(), ref, insert_hash_entry); assert(add_result.first.load_relaxed() == ref); } } } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build(vespalib::ConstArrayRef<EntryRef> refs) +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::build(vespalib::ConstArrayRef<EntryRef> refs) { - typename DictionaryType::Builder builder(_dict.getAllocator()); + typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); for (const auto& ref : refs) { builder.insert(ref, DataType()); } - _dict.assign(builder); - if constexpr (has_unordered_dictionary) { + this->_btree_dict.assign(builder); + if constexpr (has_hash_dictionary) { for (const auto& ref : refs) { - std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); - auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); + std::function<EntryRef(void)> insert_hash_entry([ref]() noexcept -> EntryRef { return ref; }); + auto& add_result = this->_hash_dict.add(this->_hash_dict.get_default_comparator(), ref, insert_hash_entry); assert(add_result.first.load_relaxed() == ref); } } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, vespalib::ConstArrayRef<uint32_t> payloads) { assert(refs.size() == payloads.size()); - typename DictionaryType::Builder builder(_dict.getAllocator()); + typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); for (size_t i = 0; i < refs.size(); ++i) { if constexpr (std::is_same_v<DataType, uint32_t>) { builder.insert(refs[i], payloads[i]); @@ -250,12 +250,12 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build_with_pa builder.insert(refs[i], DataType()); } } - _dict.assign(builder); - if constexpr (has_unordered_dictionary) { + this->_btree_dict.assign(builder); + if constexpr (has_hash_dictionary) { for (size_t i = 0; i < refs.size(); ++i) { EntryRef ref = refs[i]; - std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); - auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); + std::function<EntryRef(void)> insert_hash_entry([ref]() noexcept -> EntryRef { return ref; }); + auto& add_result = this->_hash_dict.add(this->_hash_dict.get_default_comparator(), ref, insert_hash_entry); assert(add_result.first.load_relaxed() == refs[i]); if constexpr (std::is_same_v<DataType, uint32_t>) { add_result.second.store_relaxed(EntryRef(payloads[i])); @@ -264,11 +264,18 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build_with_pa } } -template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> std::unique_ptr<typename ParentT::ReadSnapshot> -UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::get_read_snapshot() const +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::get_read_snapshot() const { - return std::make_unique<ReadSnapshotImpl>(_dict.getFrozenView()); + return std::make_unique<ReadSnapshotImpl>(this->_btree_dict.getFrozenView()); +} + +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> +bool +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::get_has_hash_dictionary() const +{ + return has_hash_dictionary; } } diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 080b7715646..17fc14d0e9e 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -63,6 +63,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT threadstackexecutor.cpp threadstackexecutorbase.cpp time.cpp + unwind_message.cpp valgrind.cpp zstdcompressor.cpp DEPENDS diff --git a/vespalib/src/vespa/vespalib/util/macro.h b/vespalib/src/vespa/vespalib/util/macro.h index 16c66880e5d..506c1a59e54 100644 --- a/vespalib/src/vespa/vespalib/util/macro.h +++ b/vespalib/src/vespa/vespalib/util/macro.h @@ -2,8 +2,6 @@ #pragma once -// indirectly tested by exception test. - /** * @def VESPA_STRINGIZE(str) * @brief convert code to string @@ -23,3 +21,8 @@ **/ #define VESPA_STRLOC vespalib::make_string("%s in %s:%d",__func__,__FILE__,__LINE__) +/** + * Create a new token by concatenating two tokens (token pasting) + **/ +#define VESPA_CAT_IMPL(a, b) a ## b +#define VESPA_CAT(a, b) VESPA_CAT_IMPL(a, b) diff --git a/vespalib/src/vespa/vespalib/util/require.cpp b/vespalib/src/vespa/vespalib/util/require.cpp index c00683c018e..170e37ef21f 100644 --- a/vespalib/src/vespa/vespalib/util/require.cpp +++ b/vespalib/src/vespa/vespalib/util/require.cpp @@ -3,18 +3,26 @@ #include "require.h" #include <vespa/vespalib/stllike/asciistream.h> #include <iostream> -#include <stdexcept> namespace vespalib { +VESPA_IMPLEMENT_EXCEPTION(RequireFailedException, Exception); + +void throw_require_failed(const char *description, const char *file, uint32_t line) +{ + asciistream msg; + msg << "error: (" << description << ") failed"; + asciistream loc; + loc << "file " << file << " line " << line; + throw RequireFailedException(msg.c_str(), loc.c_str(), 2); +} + void handle_require_failure(const char *description, const char *file, uint32_t line) { - asciistream msg; - msg << "in " << file; - msg << " line " << line; - msg << " requirement (" << description << ") fails"; - std::cerr << msg.c_str() << "\n"; - throw std::invalid_argument(msg.c_str()); + asciistream msg; + std::cerr << file << ":" << line << ": "; + std::cerr << "error: (" << description << ") failed\n"; + throw_require_failed(description, file, line); } } // namespace diff --git a/vespalib/src/vespa/vespalib/util/require.h b/vespalib/src/vespa/vespalib/util/require.h index 9de58142926..c666a06c8d6 100644 --- a/vespalib/src/vespa/vespalib/util/require.h +++ b/vespalib/src/vespa/vespalib/util/require.h @@ -2,32 +2,53 @@ #pragma once +#include "macro.h" #include <iostream> +#include <vespa/vespalib/util/exception.h> namespace vespalib { +VESPA_DEFINE_EXCEPTION(RequireFailedException, Exception); + constexpr void handle_require_success() {} + +void throw_require_failed [[noreturn]] (const char *description, const char *file, uint32_t line); + void handle_require_failure [[noreturn]] (const char *description, const char *file, uint32_t line); + template<typename A, typename B> -void handle_require_eq_failure [[noreturn]] (const A& a, const B& b, +void handle_require_eq_failure [[noreturn]] (const A& a, const B& b, const char *a_desc, const char *b_desc, const char *description, const char *file, uint32_t line) { - std::cerr << "( " << a << " == " << b << " ) is false\n"; - handle_require_failure(description, file, line); + std::cerr << file << ":" << line << ": error: "; + std::cerr << "expected (" << a_desc << " == " << b_desc << ")\n"; + std::cerr << " lhs (" << a_desc << ") is: " << a << "\n"; + std::cerr << " rhs (" << b_desc << ") is: " << b << "\n"; + throw_require_failed(description, file, line); } -#ifndef __STRING -#define __STRING(x) #x -#endif - -#define REQUIRE(...) \ - (__VA_ARGS__) ? vespalib::handle_require_success() : \ - vespalib::handle_require_failure(__STRING(__VA_ARGS__), \ - __FILE__, __LINE__) - -#define REQUIRE_EQ(a, b) \ - (a == b) ? vespalib::handle_require_success() : \ - vespalib::handle_require_eq_failure(a, b, __STRING(a) " == " __STRING(b), \ - __FILE__, __LINE__) +/** + * Require a condition to be true. + * If the requirement is not met, prints a nice message and throws + * an exception. Use instead of assert() or ASSERT_TRUE(). + **/ +#define REQUIRE(...) \ + (__VA_ARGS__) ? vespalib::handle_require_success() : \ + vespalib::handle_require_failure(VESPA_STRINGIZE(__VA_ARGS__), \ + __FILE__, __LINE__) + +/** + * Require two values to be equal. + * If the requirement is not met, prints a nice message and throws + * an exception. Use instead of assert() or ASSERT_TRUE(). + * Note: both operator== and operator<< (to stream) must be implemented + * for the value types. + **/ +#define REQUIRE_EQ(a, b) \ + (a == b) ? vespalib::handle_require_success() : \ + vespalib::handle_require_eq_failure(a, b, \ + VESPA_STRINGIZE(a), VESPA_STRINGIZE(b), \ + VESPA_STRINGIZE(a) " == " VESPA_STRINGIZE(b), \ + __FILE__, __LINE__) } // namespace diff --git a/vespalib/src/vespa/vespalib/util/unwind_message.cpp b/vespalib/src/vespa/vespalib/util/unwind_message.cpp new file mode 100644 index 00000000000..5598f0068a5 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/unwind_message.cpp @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "unwind_message.h" +#include <exception> + +namespace vespalib { + +UnwindMessage::UnwindMessage(const vespalib::string &msg) + : _num_active(std::uncaught_exceptions()), + _message(msg) +{ +} + +UnwindMessage::UnwindMessage(UnwindMessage &&rhs) + : _num_active(std::uncaught_exceptions()), + _message(rhs._message) +{ + rhs._message.clear(); +} + +UnwindMessage::~UnwindMessage() { + if ((std::uncaught_exceptions() != _num_active) && !_message.empty()) { + fprintf(stderr, "%s\n", _message.c_str()); + } +} + +UnwindMessage unwind_msg(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vespalib::string msg = make_string_va(fmt, ap); + va_end(ap); + return {msg}; +} + +} // namespace diff --git a/vespalib/src/vespa/vespalib/util/unwind_message.h b/vespalib/src/vespa/vespalib/util/unwind_message.h new file mode 100644 index 00000000000..43300ab830a --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/unwind_message.h @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "macro.h" +#include "stringfmt.h" + +namespace vespalib { + +/** + * This class contains a message that will be printed to stderr if the + * object is destructed due to stack unwinding caused by an exception. + **/ +class UnwindMessage { +private: + int _num_active; + vespalib::string _message; +public: + UnwindMessage(const vespalib::string &msg); + UnwindMessage(UnwindMessage &&rhs); + UnwindMessage(const UnwindMessage &) = delete; + UnwindMessage &operator=(const UnwindMessage &) = delete; + UnwindMessage &operator=(UnwindMessage &&) = delete; + ~UnwindMessage(); +}; + +extern UnwindMessage unwind_msg(const char *fmt, ...) +#ifdef __GNUC__ + // Add printf format checks with gcc + __attribute__ ((format (printf,1,2))) +#endif + ; + +// make an unwind message with a hopefully unique name on the stack +#define UNWIND_MSG(...) auto VESPA_CAT(unwindMessageOnLine, __LINE__) = unwind_msg(__VA_ARGS__) + +// make an unwind message quoting a piece of code and then perform that code +#define UNWIND_DO(...) do { UNWIND_MSG("%s:%d: %s", __FILE__, __LINE__, VESPA_STRINGIZE(__VA_ARGS__)); __VA_ARGS__; } while(false) + +} // namespace diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index 7563ef068a9..25cc678fe66 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -64,7 +64,7 @@ <!-- Needed to have the same version as slf4j-api --> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> - <version>1.7.5</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.apache.zookeeper</groupId> diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index c186b377eb6..50301914e41 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -36,7 +36,7 @@ <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> - <version>1.7.25</version> + <scope>compile</scope> </dependency> </dependencies> <build> diff --git a/zookeeper-server/zookeeper-server-3.6.2/pom.xml b/zookeeper-server/zookeeper-server-3.6.2/pom.xml index 04f8b012b09..2f0029169ea 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/pom.xml +++ b/zookeeper-server/zookeeper-server-3.6.2/pom.xml @@ -41,7 +41,7 @@ <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> - <version>1.7.5</version> + <version>${slf4j.version}</version> </dependency> <!-- snappy-java and metrics-core are included here to be able to work with ZooKeeper 3.6.2 due to |