diff options
438 files changed, 6567 insertions, 6092 deletions
diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 5b8bd2032d6..16dbd6b4014 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -30,7 +30,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.32.v20200930</jetty.version> + <jetty.version>9.4.35.v20201120</jetty.version> <junit5.version>5.7.0</junit5.version> <junit5.platform.version>1.7.0</junit5.platform.version> <org.lz4.version>1.7.1</org.lz4.version> @@ -259,6 +259,7 @@ <include>org.eclipse.jetty:jetty-server:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-servlet:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-servlets:[${jetty.version}]:jar:test</include> + <include>org.eclipse.jetty:jetty-util-ajax:[${jetty.version}]:jar:test</include> <include>org.hamcrest:hamcrest-core:1.3:jar:test</include> <include>org.hdrhistogram:HdrHistogram:2.1.8:jar:test</include> <include>org.junit.jupiter:junit-jupiter-api:[${junit5.version}]:jar:test</include> diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java index 2044e6869f6..202bd92d86e 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java @@ -28,14 +28,6 @@ import static java.util.stream.Collectors.toUnmodifiableMap; */ public class ReindexingCurator { - private static final String STATUS = "status"; - private static final String TYPE = "type"; - private static final String STARTED_MILLIS = "startedMillis"; - private static final String ENDED_MILLIS = "endedMillis"; - private static final String PROGRESS = "progress"; - private static final String STATE = "state"; - private static final String MESSAGE = "message"; - private final Curator curator; private final String clusterName; private final ReindexingSerializer serializer; @@ -78,6 +70,14 @@ public class ReindexingCurator { private static class ReindexingSerializer { + private static final String STATUS = "status"; + private static final String TYPE = "type"; + private static final String STARTED_MILLIS = "startedMillis"; + private static final String ENDED_MILLIS = "endedMillis"; + private static final String PROGRESS = "progress"; + private static final String STATE = "state"; + private static final String MESSAGE = "message"; + private final DocumentTypeManager types; public ReindexingSerializer(DocumentTypeManager types) { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java index a336ad02f20..7989338c406 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java @@ -56,19 +56,18 @@ public class ReindexingMaintainer extends AbstractComponent { Metric metric, DocumentAccess access, ZookeepersConfig zookeepersConfig, ClusterListConfig clusterListConfig, AllClustersBucketSpacesConfig allClustersBucketSpacesConfig, - ReindexingConfig reindexingConfig, DocumentmanagerConfig documentmanagerConfig) { - this(Clock.systemUTC(), metric, access, zookeepersConfig, clusterListConfig, allClustersBucketSpacesConfig, reindexingConfig, documentmanagerConfig); + ReindexingConfig reindexingConfig) { + this(Clock.systemUTC(), metric, access, zookeepersConfig, clusterListConfig, allClustersBucketSpacesConfig, reindexingConfig); } ReindexingMaintainer(Clock clock, Metric metric, DocumentAccess access, ZookeepersConfig zookeepersConfig, ClusterListConfig clusterListConfig, AllClustersBucketSpacesConfig allClustersBucketSpacesConfig, - ReindexingConfig reindexingConfig, DocumentmanagerConfig documentmanagerConfig) { - DocumentTypeManager manager = new DocumentTypeManager(documentmanagerConfig); - this.reindexer = new Reindexer(parseCluster(reindexingConfig.clusterName(), clusterListConfig, allClustersBucketSpacesConfig, manager), - parseReady(reindexingConfig, manager), + ReindexingConfig reindexingConfig) { + this.reindexer = new Reindexer(parseCluster(reindexingConfig.clusterName(), clusterListConfig, allClustersBucketSpacesConfig, access.getDocumentTypeManager()), + parseReady(reindexingConfig, access.getDocumentTypeManager()), new ReindexingCurator(Curator.create(zookeepersConfig.zookeeperserverlist()), reindexingConfig.clusterName(), - manager), + access.getDocumentTypeManager()), access, metric, clock); diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java new file mode 100644 index 00000000000..fca08f7743c --- /dev/null +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java @@ -0,0 +1,98 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.reindexing.http; + +import ai.vespa.reindexing.Reindexing; +import ai.vespa.reindexing.ReindexingCurator; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.Metric; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; +import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.zookeeper.VespaZooKeeperServer; + +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; + +/** + * Allows inspecting reindexing status over HTTP. + * + * @author jonmv + */ +public class ReindexingV1ApiHandler extends ThreadedHttpRequestHandler { + + private final ReindexingCurator database; + + @Inject + public ReindexingV1ApiHandler(Executor executor, Metric metric, + @SuppressWarnings("unused") VespaZooKeeperServer ensureZkHasStarted, ZookeepersConfig zookeepersConfig, + ReindexingConfig reindexingConfig, DocumentmanagerConfig documentmanagerConfig) { + this(executor, + metric, + new ReindexingCurator(Curator.create(zookeepersConfig.zookeeperserverlist()), + reindexingConfig.clusterName(), + new DocumentTypeManager(documentmanagerConfig))); + } + + ReindexingV1ApiHandler(Executor executor, Metric metric, ReindexingCurator database) { + super(executor, metric); + this.database = database; + } + + @Override + public HttpResponse handle(HttpRequest request) { + Path path = new Path(request.getUri()); + if (request.getMethod() != GET) + return ErrorResponse.methodNotAllowed("Only GET is supported under /reindexing/v1/"); + + if (path.matches("/reindexing/v1")) return getRoot(); + if (path.matches("/reindexing/v1/status")) return getStatus(); + + return ErrorResponse.notFoundError("Nothing at " + request.getUri().getRawPath()); + } + + HttpResponse getRoot() { + Slime slime = new Slime(); + slime.setObject().setArray("resources").addObject().setString("url", "/reindexing/v1/status"); + return new SlimeJsonResponse(slime); + } + + HttpResponse getStatus() { + Slime slime = new Slime(); + Cursor statusArray = slime.setObject().setArray("status"); + database.readReindexing().status().forEach((type, status) -> { + Cursor statusObject = statusArray.addObject(); + statusObject.setString("type", type.getName()); + statusObject.setLong("startedMillis", status.startedAt().toEpochMilli()); + status.endedAt().ifPresent(endedAt -> statusObject.setLong("endedMillis", endedAt.toEpochMilli())); + status.progress().ifPresent(progress -> statusObject.setString("progress", progress.serializeToString())); + statusObject.setString("state", toString(status.state())); + status.message().ifPresent(message -> statusObject.setString("message", message)); + }); + return new SlimeJsonResponse(slime); + } + + + private static String toString(Reindexing.State state) { + switch (state) { + case READY: return "pending"; + case RUNNING: return "running"; + case SUCCESSFUL: return "successful"; + case FAILED: return "failed"; + default: throw new IllegalArgumentException("Unexpected state '" + state + "'"); + } + } + +} diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java new file mode 100644 index 00000000000..1b6379d21e5 --- /dev/null +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.reindexing.http; + +import ai.vespa.reindexing.Reindexing; +import ai.vespa.reindexing.Reindexing.Status; +import ai.vespa.reindexing.ReindexingCurator; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.concurrent.Executors; + +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +class ReindexingV1ApiTest { + + DocumentmanagerConfig musicConfig = Deriver.getDocumentManagerConfig("src/test/resources/schemas/music.sd").build(); + DocumentTypeManager manager = new DocumentTypeManager(musicConfig); + DocumentType musicType = manager.getDocumentType("music"); + ReindexingCurator database = new ReindexingCurator(new MockCurator(), "cluster", manager); + ReindexingV1ApiHandler handler = new ReindexingV1ApiHandler(Executors.newSingleThreadExecutor(), new MockMetric(), database); + + @Test + void testResponses() { + RequestHandlerTestDriver driver = new RequestHandlerTestDriver(handler); + + // GET at root + var response = driver.sendRequest("http://localhost/reindexing/v1/"); + assertEquals("{\"resources\":[{\"url\":\"/reindexing/v1/status\"}]}", response.readAll()); + assertEquals("application/json; charset=UTF-8", response.getResponse().headers().getFirst("Content-Type")); + assertEquals(200, response.getStatus()); + + // GET at status with empty database + response = driver.sendRequest("http://localhost/reindexing/v1/status"); + assertEquals("{\"status\":[]}", response.readAll()); + assertEquals(200, response.getStatus()); + + // GET at status with a failed status + database.writeReindexing(Reindexing.empty().with(musicType, Status.ready(Instant.EPOCH) + .running() + .progressed(new ProgressToken()) + .failed(Instant.ofEpochMilli(123), "ヽ(。_°)ノ"))); + response = driver.sendRequest("http://localhost/reindexing/v1/status"); + assertEquals("{\"status\":[{" + + "\"type\":\"music\"," + + "\"startedMillis\":0," + + "\"endedMillis\":123," + + "\"progress\":\"" + new ProgressToken().serializeToString() + "\"," + + "\"state\":\"failed\"," + + "\"message\":\"ヽ(。_°)ノ\"}" + + "]}", + response.readAll()); + assertEquals(200, response.getStatus()); + + // POST at root + response = driver.sendRequest("http://localhost/reindexing/v1/status", POST); + assertEquals("{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Only GET is supported under /reindexing/v1/\"}", + response.readAll()); + assertEquals(405, response.getStatus()); + + // GET at non-existent path + response = driver.sendRequest("http://localhost/reindexing/v1/moo"); + assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at /reindexing/v1/moo\"}", + response.readAll()); + assertEquals(404, response.getStatus()); + + } + +} diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 9df68821454..fd229b35778 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -529,15 +529,14 @@ "public static final enum com.yahoo.config.application.api.ValidationId indexModeChange", "public static final enum com.yahoo.config.application.api.ValidationId fieldTypeChange", "public static final enum com.yahoo.config.application.api.ValidationId clusterSizeReduction", + "public static final enum com.yahoo.config.application.api.ValidationId tensorTypeChange", "public static final enum com.yahoo.config.application.api.ValidationId resourcesReduction", "public static final enum com.yahoo.config.application.api.ValidationId contentTypeRemoval", "public static final enum com.yahoo.config.application.api.ValidationId contentClusterRemoval", "public static final enum com.yahoo.config.application.api.ValidationId deploymentRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId skipAutomaticTenantUpgradeTests", "public static final enum com.yahoo.config.application.api.ValidationId globalDocumentChange", "public static final enum com.yahoo.config.application.api.ValidationId configModelVersionMismatch", "public static final enum com.yahoo.config.application.api.ValidationId skipOldConfigModels", - "public static final enum com.yahoo.config.application.api.ValidationId forceAutomaticTenantUpgradeTests", "public static final enum com.yahoo.config.application.api.ValidationId accessControl", "public static final enum com.yahoo.config.application.api.ValidationId globalEndpointChange" ] @@ -577,10 +576,7 @@ "attributes": [ "public" ], - "methods": [ - "public com.yahoo.config.application.api.ValidationId validationId()", - "public java.lang.String getMessage()" - ], + "methods": [], "fields": [] }, "com.yahoo.config.application.api.ValidationOverrides": { @@ -591,6 +587,7 @@ ], "methods": [ "public void <init>(java.util.List)", + "public void invalid(java.util.Map, java.time.Instant)", "public void invalid(com.yahoo.config.application.api.ValidationId, java.lang.String, java.time.Instant)", "public boolean allows(java.lang.String, java.time.Instant)", "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java index c0bae137b0d..4c76d42a17e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java @@ -14,15 +14,14 @@ public enum ValidationId { indexModeChange("indexing-mode-change"), // Changing the index mode (streaming, indexed, store-only) of documents fieldTypeChange("field-type-change"), // Field type changes clusterSizeReduction("cluster-size-reduction"), // Large reductions in cluster size + tensorTypeChange("tensor-type-change"), // Tensor type change resourcesReduction("resources-reduction"), // Large reductions in node resources contentTypeRemoval("content-type-removal"), // Removal of a data type (causes deletion of all data) contentClusterRemoval("content-cluster-removal"), // Removal (or id change) of content clusters deploymentRemoval("deployment-removal"), // Removal of production zones from deployment.xml - skipAutomaticTenantUpgradeTests("skip-automatic-tenant-upgrade-test"), // Skip platform supplied staging tests globalDocumentChange("global-document-change"), // Changing global attribute for document types in content clusters configModelVersionMismatch("config-model-version-mismatch"), // Internal use skipOldConfigModels("skip-old-config-models"), // Internal use - forceAutomaticTenantUpgradeTests("force-automatic-tenant-upgrade-test"), // Internal use accessControl("access-control"), // Internal use, used in zones where there should be no access-control globalEndpointChange("global-endpoint-change"); // Changing global endpoints diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index a85e109f731..f22cf0d8c47 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -14,9 +14,13 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.logging.Level; +import java.util.stream.Collectors; /** * A set of allows which suppresses specific validations in limited time periods. @@ -47,6 +51,14 @@ public class ValidationOverrides { this.xmlForm = xmlForm; } + /** Throws a ValidationException unless all given validation is overridden at this time */ + public void invalid(Map<ValidationId, ? extends Collection<String>> messagesByValidationId, Instant now) { + Map<ValidationId, Collection<String>> disallowed = new HashMap<>(messagesByValidationId); + disallowed.keySet().removeIf(id -> allows(id, now)); + if ( ! disallowed.isEmpty()) + throw new ValidationException(disallowed); + } + /** Throws a ValidationException unless this validation is overridden at this time */ public void invalid(ValidationId validationId, String message, Instant now) { if ( ! allows(validationId, now)) @@ -146,26 +158,21 @@ public class ValidationOverrides { /** * A deployment validation exception. * Deployment validations can be {@link ValidationOverrides overridden} based on their id. - * The purpose of this exception is to model that id as a separate field. */ public static class ValidationException extends IllegalArgumentException { static final long serialVersionUID = 789984668; - private final ValidationId validationId; - private ValidationException(ValidationId validationId, String message) { - super(message); - this.validationId = validationId; + super(validationId + ": " + message + ". " + toAllowMessage(validationId)); } - /** Returns the unique id of this validation, which can be used to {@link ValidationOverrides override} it */ - public ValidationId validationId() { return validationId; } - - /** Returns "validationId: message" */ - @Override - public String getMessage() { - return validationId + ": " + super.getMessage() + ". " + toAllowMessage(validationId); + private ValidationException(Map<ValidationId, Collection<String>> messagesById) { + super(messagesById.entrySet().stream() + .map(messages -> messages.getKey() + ":\n\t" + + String.join("\n\t", messages.getValue()) + "\n" + + toAllowMessage(messages.getKey())) + .collect(Collectors.joining("\n"))); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java index 1248560c931..87a150a6c3c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java @@ -1,9 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import java.util.List; +import java.util.Optional; /** * Contains the action to be performed on the given services to handle a config change @@ -37,12 +39,11 @@ public interface ConfigChangeAction { /** Returns the list of services where the action must be performed */ List<ServiceInfo> getServices(); - /** Returns whether this change should be allowed */ - boolean allowed(); + /** When this is non-empty, validation may fail unless this validation id is allowed by validation overrides. */ + default Optional<ValidationId> validationId() { return Optional.empty(); } /** The id of the cluster that needs this action applied */ - // TODO: Remove this default implementation after October 2020 - default ClusterSpec.Id clusterId() { return null; } + ClusterSpec.Id clusterId(); /** Returns whether this change should be ignored for internal redeploy */ default boolean ignoreForInternalRedeploy() { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java index c5a7bd030e2..13109088dcd 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java @@ -1,6 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.application.api.ValidationId; + /** * Represents an action to re-feed a document type in order to handle a config change. * @@ -12,7 +14,7 @@ public interface ConfigChangeRefeedAction extends ConfigChangeAction { default Type getType() { return Type.REFEED; } /** Returns the name identifying this kind of change, used to identify names which should be allowed */ - String name(); + default String name() { return validationId().orElseThrow().value(); } /** Returns the name of the document type that one must re-feed to handle this config change */ String getDocumentType(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java index 085638e31ff..bb714a55f94 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java @@ -1,6 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.application.api.ValidationId; + /** * Represents an action to re-index a document type in order to handle a config change. * @@ -11,7 +13,7 @@ public interface ConfigChangeReindexAction extends ConfigChangeAction { @Override default Type getType() { return Type.REINDEX; } /** @return name identifying this kind of change, used to identify names which should be allowed */ - String name(); + default String name() { return validationId().orElseThrow().value(); } /** @return name of the document type that must be re-indexed */ String getDocumentType(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java index f178180b6e0..c13399a42f5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java @@ -11,8 +11,4 @@ public interface ConfigChangeRestartAction extends ConfigChangeAction { @Override default Type getType() { return Type.RESTART; } - /** Restarts are handled automatically so they are allowed */ - @Override - default boolean allowed() { return true; } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index acd4d705889..2ffc24239f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -97,7 +97,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private static final long serialVersionUID = 1L; - public static final Logger log = Logger.getLogger(VespaModel.class.getPackage().toString()); + public static final Logger log = Logger.getLogger(VespaModel.class.getName()); private final Version version; private final ConfigModelRepo configModelRepo = new ConfigModelRepo(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index ec8607daaca..9e15db348a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -3,13 +3,16 @@ package com.yahoo.vespa.model.admin.clustercontroller; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.documentapi.DocumentAccessProvider; import com.yahoo.container.di.config.PlatformBundlesConfig; +import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; @@ -28,12 +31,15 @@ import java.util.TreeSet; @RestartConfigs({FleetcontrollerConfig.class, ZookeeperServerConfig.class}) public class ClusterControllerContainer extends Container implements PlatformBundlesConfig.Producer, - ZookeeperServerConfig.Producer + ZookeeperServerConfig.Producer, + ReindexingConfig.Producer { private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); private static final ComponentSpecification ZOOKEEPER_SERVER_BUNDLE = new ComponentSpecification("zookeeper-server"); + private static final ComponentSpecification REINDEXING_CONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-reindexer"); private final Set<String> bundles = new TreeSet<>(); + private final ReindexingContext reindexingContext; public ClusterControllerContainer( AbstractConfigProducer<?> parent, @@ -42,12 +48,16 @@ public class ClusterControllerContainer extends Container implements boolean isHosted, ReindexingContext reindexingContext) { super(parent, "" + index, index, isHosted); + this.reindexingContext = reindexingContext; + addHandler("clustercontroller-status", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler", - "/clustercontroller-status/*"); + "/clustercontroller-status/*", + CLUSTERCONTROLLER_BUNDLE); addHandler("clustercontroller-state-restapi-v2", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", - "/cluster/v2/*"); + "/cluster/v2/*", + CLUSTERCONTROLLER_BUNDLE); if (runStandaloneZooKeeper) { addComponent("clustercontroller-zkrunner", "com.yahoo.vespa.zookeeper.VespaZooKeeperServerImpl", @@ -73,7 +83,7 @@ public class ClusterControllerContainer extends Container implements addFileBundle("clustercontroller-core"); addFileBundle("clustercontroller-utils"); addFileBundle("zookeeper-server"); - configureReindexing(reindexingContext); + configureReindexing(); } @Override @@ -110,15 +120,21 @@ public class ClusterControllerContainer extends Container implements addComponent(new Component<>(createComponentModel(id, className, bundle))); } - private void addHandler(String id, String className, String path) { - addHandler(new Handler(createComponentModel(id, className, CLUSTERCONTROLLER_BUNDLE)), path); + private void addHandler(String id, String className, String path, ComponentSpecification bundle) { + addHandler(new Handler(createComponentModel(id, className, bundle)), path); } - private void configureReindexing(ReindexingContext context) { - if (context != null) { - addFileBundle(ReindexingController.REINDEXING_CONTROLLER_BUNDLE); - addComponent(new ReindexingController(context)); + private void configureReindexing() { + if (reindexingContext != null) { + addFileBundle(REINDEXING_CONTROLLER_BUNDLE.getName()); addComponent(new SimpleComponent(DocumentAccessProvider.class.getName())); + addComponent("reindexing-maintainer", + "ai.vespa.reindexing.ReindexingMaintainer", + REINDEXING_CONTROLLER_BUNDLE); + addHandler("reindexing-status", + "ai.vespa.reindexing.http.ReindexingV1ApiHandler", + "/reindexing/v1/*", + REINDEXING_CONTROLLER_BUNDLE); } } @@ -133,4 +149,20 @@ public class ClusterControllerContainer extends Container implements builder.myid(index()); } + @Override + public void getConfig(ReindexingConfig.Builder builder) { + if (reindexingContext == null) + return; + + builder.clusterName(reindexingContext.contentClusterName()); + builder.enabled(reindexingContext.reindexing().enabled()); + for (NewDocumentType type : reindexingContext.documentTypes()) { + String typeName = type.getFullName().getName(); + reindexingContext.reindexing().status(reindexingContext.contentClusterName(), typeName) + .ifPresent(status -> builder.status(typeName, + new ReindexingConfig.Status.Builder() + .readyAtMillis(status.ready().toEpochMilli()))); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingController.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingController.java deleted file mode 100644 index 24909ddbc8d..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingController.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.clustercontroller; - -import com.yahoo.config.model.api.Reindexing; -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; -import com.yahoo.vespa.model.container.component.SimpleComponent; - -import java.util.Collection; - -/** - * @author bjorncs - */ -class ReindexingController extends SimpleComponent implements ReindexingConfig.Producer { - - static final String REINDEXING_CONTROLLER_BUNDLE = "clustercontroller-reindexer"; - - private final Reindexing reindexing; - private final String contentClusterName; - private final Collection<NewDocumentType> documentTypes; - - ReindexingController(ReindexingContext context) { - super(new ComponentModel( - BundleInstantiationSpecification.getFromStrings( - "reindexing-maintainer", - "ai.vespa.reindexing.ReindexingMaintainer", - REINDEXING_CONTROLLER_BUNDLE))); - this.reindexing = context.reindexing(); - this.contentClusterName = context.contentClusterName(); - this.documentTypes = context.documentTypes(); - } - - @Override - public void getConfig(ReindexingConfig.Builder builder) { - builder.clusterName(contentClusterName); - builder.enabled(reindexing.enabled()); - for (NewDocumentType type : documentTypes) { - String typeName = type.getFullName().getName(); - reindexing.status(contentClusterName, typeName).ifPresent(status -> - builder.status( - typeName, - new ReindexingConfig.Status.Builder() - .readyAtMillis(status.ready().toEpochMilli()))); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 21f323fc0f3..4317f947e12 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -239,6 +239,8 @@ public class VespaMetricSet { // DO NOT RELY ON THIS METRIC YET. metrics.add(new Metric("cluster-controller.node-event.count")); + metrics.add(new Metric("cluster-controller.reindexing.progress.last")); + return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index d300e31c3dc..e028eaea3c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.Model; @@ -26,13 +27,20 @@ import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDe import java.time.Instant; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; /** * Executor of validators. This defines the right order of validator execution. @@ -99,9 +107,17 @@ public class Validation { new ContainerRestartValidator(), new NodeResourceChangeValidator() }; - return Arrays.stream(validators) - .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) - .collect(toList()); + List<ConfigChangeAction> actions = Arrays.stream(validators) + .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) + .collect(toList()); + + Map<ValidationId, Collection<String>> disallowableActions = actions.stream() + .filter(action -> action.validationId().isPresent()) + .collect(groupingBy(action -> action.validationId().orElseThrow(), + mapping(ConfigChangeAction::getMessage, + toCollection(LinkedHashSet::new)))); + overrides.invalid(disallowableActions, now); + return actions; } private static void validateFirstTimeDeployment(VespaModel model, DeployState deployState) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java index b321d5f3fd7..58cea8c23e5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -13,7 +13,10 @@ import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; import java.time.Instant; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index e3f16baf95a..385a678d452 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -46,22 +46,22 @@ public class IndexingModeChangeValidator implements ChangeValidator { ContentSearchCluster currentSearchCluster = currentCluster.getSearch(); ContentSearchCluster nextSearchCluster = nextCluster.getSearch(); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithStreamingCluster()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithIndexedCluster()), "streaming", "indexed"); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithIndexedCluster()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithStreamingCluster()), "indexed", "streaming"); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithStoreOnly()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithIndexedCluster()), "store-only", "indexed"); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithIndexedCluster()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithStoreOnly()), "indexed", "store-only"); @@ -69,7 +69,7 @@ public class IndexingModeChangeValidator implements ChangeValidator { } private static void findDocumentTypesWithActionableIndexingModeChange( - List<ConfigChangeAction> actions, ValidationOverrides overrides, ContentCluster nextCluster, Instant now, + List<ConfigChangeAction> actions, ContentCluster nextCluster, Set<String> currentTypes, Set<String> nextTypes, String currentIndexMode, String nextIndexingMode) { for (String type : nextTypes) { if (currentTypes.contains(type)) { @@ -78,14 +78,13 @@ public class IndexingModeChangeValidator implements ChangeValidator { .collect(Collectors.toList()); actions.add(VespaReindexAction.of( nextCluster.id(), - ValidationId.indexModeChange.value(), - overrides, + ValidationId.indexModeChange, String.format( "Document type '%s' in cluster '%s' changed indexing mode from '%s' to '%s'", type, nextCluster.getName(), currentIndexMode, nextIndexingMode), services, - type, - now)); + type + )); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index d85d9bd2db5..2f13caa4e09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -78,7 +78,7 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { NewDocumentType nextDocType, ValidationOverrides overrides, Instant now) { - return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate(overrides, now); + return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate(); } private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java index 19c63431c03..6a335447a31 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.application.api.ValidationOverrides; @@ -8,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import java.time.Instant; import java.util.List; +import java.util.Optional; /** * Represents an action to re-feed a document type in order to handle a config change. @@ -22,45 +24,39 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config * the validation ids belong to the Vespa model while these names are exposed to the config server, * which is model version independent. */ - private final String name; - + private final ValidationId validationId; private final String documentType; - private final boolean allowed; - private VespaRefeedAction(ClusterSpec.Id id, String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { + private VespaRefeedAction(ClusterSpec.Id id, ValidationId validationId, String message, List<ServiceInfo> services, String documentType) { super(id, message, services); - this.name = name; + this.validationId = validationId; this.documentType = documentType; - this.allowed = allowed; } /** Creates a refeed action with some missing information */ // TODO: We should require document type or model its absence properly - public static VespaRefeedAction of(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return new VespaRefeedAction(id, name, message, List.of(), "", overrides.allows(name, now)); + public static VespaRefeedAction of(ClusterSpec.Id id, ValidationId validationId, String message) { + return new VespaRefeedAction(id, validationId, message, List.of(), ""); } /** Creates a refeed action */ - public static VespaRefeedAction of(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return new VespaRefeedAction(id, name, message, services, documentType, overrides.allows(name, now)); + public static VespaRefeedAction of(ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return new VespaRefeedAction(id, validationId, message, services, documentType); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaRefeedAction(clusterId(), name, newMessage, newServices, documentType, allowed); + return new VespaRefeedAction(clusterId(), validationId, newMessage, newServices, documentType); } @Override - public String name() { return name; } + public Optional<ValidationId> validationId() { return Optional.of(validationId); } @Override public String getDocumentType() { return documentType; } @Override - public boolean allowed() { return allowed; } - - @Override public boolean ignoreForInternalRedeploy() { return false; } @@ -76,14 +72,13 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config if ( ! (o instanceof VespaRefeedAction)) return false; VespaRefeedAction other = (VespaRefeedAction)o; if ( ! this.documentType.equals(other.documentType)) return false; - if ( ! this.name.equals(other.name)) return false; - if ( ! this.allowed == other.allowed) return false; + if ( ! this.validationId.equals(other.validationId)) return false; return true; } @Override public int hashCode() { - return 31 * super.hashCode() + 11 * name.hashCode() + documentType.hashCode(); + return 31 * super.hashCode() + 11 * validationId.hashCode() + documentType.hashCode(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java index f10802afc31..8b4060e7d19 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java @@ -1,14 +1,14 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; -import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeReindexAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; -import java.time.Instant; import java.util.List; import java.util.Objects; +import java.util.Optional; /** * Represents an action to re-index a document type in order to handle a config change. @@ -22,36 +22,32 @@ public class VespaReindexAction extends VespaConfigChangeAction implements Confi * the validation ids belong to the Vespa model while these names are exposed to the config server, * which is model version independent. */ - private final String name; + private final ValidationId validationId; private final String documentType; - private final boolean allowed; - private VespaReindexAction(ClusterSpec.Id id, String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { + private VespaReindexAction(ClusterSpec.Id id, ValidationId validationId, String message, List<ServiceInfo> services, String documentType) { super(id, message, services); - this.name = name; + this.validationId = validationId; this.documentType = documentType; - this.allowed = allowed; } - public static VespaReindexAction of( - ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return new VespaReindexAction(id, name, message, List.of(), /*documentType*/null, overrides.allows(name, now)); + public static VespaReindexAction of(ClusterSpec.Id id, ValidationId validationId, String message) { + return new VespaReindexAction(id, validationId, message, List.of(), /*documentType*/null); } public static VespaReindexAction of( - ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return new VespaReindexAction(id, name, message, services, documentType, overrides.allows(name, now)); + ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return new VespaReindexAction(id, validationId, message, services, documentType); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaReindexAction(clusterId(), name, newMessage, newServices, documentType, allowed); + return new VespaReindexAction(clusterId(), validationId, newMessage, newServices, documentType); } - @Override public String name() { return name; } + @Override public Optional<ValidationId> validationId() { return Optional.of(validationId); } @Override public String getDocumentType() { return documentType; } - @Override public boolean allowed() { return allowed; } @Override public boolean ignoreForInternalRedeploy() { return false; } @Override public String toString() { return super.toString() + ", documentType='" + documentType + "'"; } @@ -61,10 +57,10 @@ public class VespaReindexAction extends VespaConfigChangeAction implements Confi if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; VespaReindexAction that = (VespaReindexAction) o; - return allowed == that.allowed && - Objects.equals(name, that.name) && - Objects.equals(documentType, that.documentType); + return Objects.equals(validationId, that.validationId) && + Objects.equals(documentType, that.documentType); } - @Override public int hashCode() { return Objects.hash(super.hashCode(), name, documentType, allowed); } + @Override public int hashCode() { return Objects.hash(super.hashCode(), validationId, documentType); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java index 3dcfbe3629d..ce435a4c157 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java @@ -72,7 +72,7 @@ public class DocumentDatabaseChangeValidator { private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides, Instant now) { return new DocumentTypeChangeValidator(id, currentDocType, nextDocType) - .validate(overrides, now); + .validate(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java index b66145a10c5..5b7fdfad0f7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java @@ -1,15 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.document.Field; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; -import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -136,17 +135,16 @@ public class DocumentTypeChangeValidator { this.nextDocType = nextDocType; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { + public List<VespaConfigChangeAction> validate() { return currentDocType.getAllFields().stream(). map(field -> createFieldChange(field, nextDocType)). filter(fieldChange -> fieldChange.valid() && fieldChange.changedType()). map(fieldChange -> VespaRefeedAction.of(id, - "field-type-change", - overrides, + ValidationId.fieldTypeChange, new ChangeMessageBuilder(fieldChange.fieldName()). addChange("data type", fieldChange.currentTypeName(), - fieldChange.nextTypeName()).build(), - now)). + fieldChange.nextTypeName()).build() + )). collect(Collectors.toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index 8f9b1a3ed77..e3f3abf0747 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -41,21 +41,20 @@ public class IndexingScriptChangeValidator { String fieldName = nextField.getName(); ImmutableSDField currentField = currentSearch.getConcreteField(fieldName); if (currentField != null) { - validateScripts(currentField, nextField, overrides, now).ifPresent(r -> result.add(r)); + validateScripts(currentField, nextField).ifPresent(r -> result.add(r)); } } return result; } - private Optional<VespaConfigChangeAction> validateScripts(ImmutableSDField currentField, ImmutableSDField nextField, - ValidationOverrides overrides, Instant now) { + private Optional<VespaConfigChangeAction> validateScripts(ImmutableSDField currentField, ImmutableSDField nextField) { ScriptExpression currentScript = currentField.getIndexingScript(); ScriptExpression nextScript = nextField.getIndexingScript(); if ( ! equalScripts(currentScript, nextScript)) { ChangeMessageBuilder messageBuilder = new ChangeMessageBuilder(nextField.getName()); new IndexingScriptChangeMessageBuilder(currentSearch, currentField, nextSearch, nextField).populate(messageBuilder); messageBuilder.addChange("indexing script", currentScript.toString(), nextScript.toString()); - return Optional.of(VespaReindexAction.of(id, ValidationId.indexingChange.value(), overrides, messageBuilder.build(), now)); + return Optional.of(VespaReindexAction.of(id, ValidationId.indexingChange, messageBuilder.build())); } return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 9bd12350f26..00ff19ae68a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -3,6 +3,10 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.BindingPattern; @@ -27,11 +31,13 @@ import java.util.Set; */ public class AccessControl { - public enum ClientAuthentication { want, need } + + public enum ClientAuthentication { want, need;} public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); - public static final ComponentId ACCESS_CONTROL_EXCLUDED_CHAIN_ID = ComponentId.fromString("access-control-excluded-chain"); + public static final ComponentId ACCESS_CONTROL_EXCLUDED_CHAIN_ID = ComponentId.fromString("access-control-excluded-chain"); + public static final ComponentId DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID = ComponentId.fromString("default-connector-hosted-request-chain"); private static final int HOSTED_CONTAINER_PORT = 4443; // Handlers that are excluded from access control @@ -44,7 +50,6 @@ public class AccessControl { ApplicationContainerCluster.METRICS_V2_HANDLER_CLASS, ApplicationContainerCluster.PROMETHEUS_V1_HANDLER_CLASS ); - public static class Builder { private final String domain; private boolean readEnabled = false; @@ -52,7 +57,6 @@ public class AccessControl { private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); private Collection<Handler<?>> handlers = Collections.emptyList(); - public Builder(String domain) { this.domain = domain; } @@ -112,6 +116,7 @@ public class AccessControl { http.setAccessControl(this); addAccessControlFilterChain(http); addAccessControlExcludedChain(http); + addDefaultHostedRequestChain(http); removeDuplicateBindingsFromAccessControlChain(http); } @@ -119,6 +124,18 @@ public class AccessControl { connectorFactory.setDefaultRequestFilterChain(ACCESS_CONTROL_CHAIN_ID); } + public void configureDefaultHostedConnector(Http http) { + // Set default filter chain on local port + http.getHttpServer() + .get() + .getConnectorFactories() + .stream() + .filter(cf -> cf.getListenPort() == Defaults.getDefaults().vespaWebServicePort()) + .findFirst() + .orElseThrow(() -> new RuntimeException("Could not find default connector")) + .setDefaultRequestFilterChain(DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID); + } + /** returns the excluded bindings as specified in 'access-control' in services.xml **/ public Set<BindingPattern> excludedBindings() { return excludedBindings; } @@ -148,6 +165,12 @@ public class AccessControl { } } + // Add a filter chain used by default hosted connector + private void addDefaultHostedRequestChain(Http http) { + Chain<Filter> chain = createChain(DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID); + http.getFilterChains().add(chain); + } + // Remove bindings from access control chain that have binding pattern as a different filter chain private void removeDuplicateBindingsFromAccessControlChain(Http http) { removeDuplicateBindingsFromChain(http, ACCESS_CONTROL_EXCLUDED_CHAIN_ID); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 0e23527c97c..7eea5d8496f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -85,6 +85,7 @@ import org.w3c.dom.Node; import java.net.URI; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -318,10 +319,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (isHostedTenantApplication(context)) { addHostedImplicitHttpIfNotPresent(cluster); addHostedImplicitAccessControlIfNotPresent(deployState, cluster); + addDefaultConnectorHostedFilterBinding(cluster); addAdditionalHostedConnector(deployState, cluster, context); } } + private void addDefaultConnectorHostedFilterBinding(ApplicationContainerCluster cluster) { + cluster.getHttp().getAccessControl() + .ifPresent(accessControl -> accessControl.configureDefaultHostedConnector(cluster.getHttp())); ; + } + private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster, ConfigModelContext context) { JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); @@ -361,10 +368,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if(cluster.getHttp() == null) { cluster.setHttp(new Http(new FilterChains(cluster))); } - if(cluster.getHttp().getHttpServer().isEmpty()) { - JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), cluster, cluster.isHostedVespa()); - cluster.getHttp().setHttpServer(defaultHttpServer); - defaultHttpServer.addConnector(new ConnectorFactory.Builder("SearchServer", Defaults.getDefaults().vespaWebServicePort()).build()); + JettyHttpServer httpServer = cluster.getHttp().getHttpServer().orElse(null); + if (httpServer == null) { + httpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), cluster, cluster.isHostedVespa()); + cluster.getHttp().setHttpServer(httpServer); + } + int defaultPort = Defaults.getDefaults().vespaWebServicePort(); + boolean defaultConnectorPresent = httpServer.getConnectorFactories().stream().anyMatch(connector -> connector.getListenPort() == defaultPort); + if (!defaultConnectorPresent) { + httpServer.addConnector(new ConnectorFactory.Builder("SearchServer", defaultPort).build()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index f3f9022f6f0..2157839ef5c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import java.time.Instant; @@ -24,26 +24,22 @@ public class ConfigChangeTestUtils { return new VespaRestartAction(id, message, services); } - public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, String message) { - return VespaRefeedAction.of(id, name, ValidationOverrides.empty, message, Instant.now()); + public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, ValidationId validationId, String message) { + return VespaRefeedAction.of(id, validationId, message); } - public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return VespaRefeedAction.of(id, name, overrides, message, now); + public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return VespaRefeedAction.of(id, validationId, message, services, documentType); } - public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return VespaRefeedAction.of(id, name, overrides, message, services, documentType, now); + public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, ValidationId validationId, String message) { + return VespaReindexAction.of(id, validationId, message); } - public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return VespaReindexAction.of(id, name, overrides, message, now); - } - - public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return VespaReindexAction.of(id, name, overrides, message, services, documentType, now); + public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return VespaReindexAction.of(id, validationId, message, services, documentType); } public static List<ConfigChangeAction> normalizeServicesInActions(List<ConfigChangeAction> result) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java index 8d365f24c7f..2b211c561d9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; @@ -146,9 +147,8 @@ public class IndexedSearchClusterChangeValidatorTest { public void requireThatChangingFieldTypeIsDiscovered() { Fixture f = Fixture.newOneDocFixture(STRING_FIELD, INT_FIELD); f.assertValidation(List.of(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Document type 'd1': " + FIELD_TYPE_CHANGE_MSG, FOO_SERVICE, "d1", Instant.now()))); + ValidationId.fieldTypeChange, + "Document type 'd1': " + FIELD_TYPE_CHANGE_MSG, FOO_SERVICE, "d1"))); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index 8e2cecf89b4..ba9dfcdc388 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationOverrides.ValidationException; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigChangeReindexAction; import com.yahoo.config.provision.Environment; @@ -11,8 +12,10 @@ import org.junit.Test; import java.util.List; import java.util.stream.Collectors; +import static java.util.stream.Collectors.joining; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author bratseth @@ -21,6 +24,25 @@ import static org.junit.Assert.assertTrue; public class IndexingModeChangeValidatorTest { @Test + public void testChangingIndexModeFromIndexedToStreamingWhenDisallowed() { + ValidationTester tester = new ValidationTester(); + + VespaModel oldModel = + tester.deploy(null, getServices("index"), Environment.prod, "<validation-overrides />").getFirst(); + try { + List<ConfigChangeAction> changeActions = + tester.deploy(oldModel, getServices("streaming"), Environment.prod, "<calidation-overrides />").getSecond(); + fail("Should throw on disallowed config change action"); + } + catch (ValidationException e) { + assertEquals("indexing-mode-change:\n" + + "\tDocument type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'\n" + + "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/documentation/reference/validation-overrides.html", + e.getMessage()); + } + } + + @Test public void testChangingIndexModeFromIndexedToStreaming() { ValidationTester tester = new ValidationTester(); @@ -29,9 +51,9 @@ public class IndexingModeChangeValidatorTest { List<ConfigChangeAction> changeActions = tester.deploy(oldModel, getServices("streaming"), Environment.prod, validationOverrides).getSecond(); - assertReindexingChange(true, // allowed=true due to validation override - "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'", - changeActions); + assertReindexingChange( // allowed=true due to validation override + "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'", + changeActions); } @Test @@ -43,23 +65,22 @@ public class IndexingModeChangeValidatorTest { List<ConfigChangeAction> changeActions = tester.deploy(oldModel, getServices("store-only"), Environment.prod, validationOverrides).getSecond(); - assertReindexingChange(true, // allowed=true due to validation override - "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'store-only'", - changeActions); + assertReindexingChange( // allowed=true due to validation override + "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'store-only'", + changeActions); } - private void assertReindexingChange(boolean allowed, String message, List<ConfigChangeAction> changeActions) { + private void assertReindexingChange(String message, List<ConfigChangeAction> changeActions) { List<ConfigChangeAction> reindexingActions = changeActions.stream() .filter(a -> a instanceof ConfigChangeReindexAction) .collect(Collectors.toList()); assertEquals(1, reindexingActions.size()); - assertEquals(allowed, reindexingActions.get(0).allowed()); assertTrue(reindexingActions.get(0) instanceof ConfigChangeReindexAction); assertEquals("indexing-mode-change", ((ConfigChangeReindexAction)reindexingActions.get(0)).name()); assertEquals(message, reindexingActions.get(0).getMessage()); } - private static final String getServices(String indexingMode) { + private static String getServices(String indexingMode) { return "<services version='1.0'>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java index f5ef50ee3a4..18aac032fe7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; @@ -164,10 +165,9 @@ public class StreamingSearchClusterChangeValidatorTest { private static VespaConfigChangeAction createFieldTypeChangeRefeedAction(String docType, List<ServiceInfo> service) { return ConfigChangeTestUtils.newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Document type '" + docType + "': Field 'f1' changed: data type: 'string' -> 'int'", - service, docType, Instant.now()); + ValidationId.fieldTypeChange, + "Document type '" + docType + "': Field 'f1' changed: data type: 'string' -> 'int'", + service, docType); } private static VespaConfigChangeAction createAddFastAccessRestartAction() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java index a4fbf474a7f..1f64d41e371 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; @@ -47,20 +48,17 @@ public class DocumentDatabaseChangeValidatorTest { "field f2 type string { indexing: index | summary } " + "field f3 type string { indexing: summary } " + "field f4 type array<s> { struct-field s1 { indexing: attribute } }"); + Instant.now(); f.assertValidation(Arrays.asList( newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1' changed: add attribute aspect"), newRestartAction(ClusterSpec.Id.from("test"), "Field 'f4.s1' changed: add attribute aspect"), newReindexAction(ClusterSpec.Id.from("test"), - "indexing-change", - ValidationOverrides.empty, - "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + - "'{ input f2 | tokenize normalize stem:\"BEST\" | index f2 | summary f2; }'", Instant.now()), - newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f3' changed: data type: 'int' -> 'string'", Instant.now()))); + ValidationId.indexingChange, + "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + + "'{ input f2 | tokenize normalize stem:\"BEST\" | index f2 | summary f2; }'"), + newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f3' changed: data type: 'int' -> 'string'"))); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index a074f961a53..8ee2a924503 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; @@ -8,7 +9,6 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; import org.junit.Test; @@ -42,7 +42,7 @@ public class DocumentTypeChangeValidatorTest { @Override public List<VespaConfigChangeAction> validate() { - return validator.validate(ValidationOverrides.empty, Instant.now()); + return validator.validate(); } } @@ -65,21 +65,16 @@ public class DocumentTypeChangeValidatorTest { public void requireThatDataTypeChangeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary }", "field f1 type int { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'string' -> 'int'", - Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'int'")); } @Test public void requireThatAddingCollectionTypeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary }", "field f1 type array<string> { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'string' -> 'Array<string>'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'Array<string>'")); } @@ -94,34 +89,26 @@ public class DocumentTypeChangeValidatorTest { public void requireThatNestedDataTypeChangeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", "field f1 type array<int> { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'Array<string>' -> 'Array<int>'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'Array<string>' -> 'Array<int>'")); } @Test public void requireThatChangedCollectionTypeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", "field f1 type weightedset<string> { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'Array<string>' -> 'WeightedSet<string>'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'Array<string>' -> 'WeightedSet<string>'")); } @Test public void requireThatMultipleDataTypeChangesIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary } field f2 type int { indexing: summary }" , "field f2 type string { indexing: summary } field f1 type int { indexing: summary }"); - f.assertValidation(Arrays.asList(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'string' -> 'int'", Instant.now()), - newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f2' changed: data type: 'int' -> 'string'", Instant.now()))); + Instant.now(); + Instant.now(); + f.assertValidation(Arrays.asList(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'int'"), + newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f2' changed: data type: 'int' -> 'string'"))); } @Test @@ -156,40 +143,32 @@ public class DocumentTypeChangeValidatorTest { public void requireThatDataTypeChangeInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} } field f2 type s1 { indexing: summary }", "struct s1 { field f1 type int {} } field f2 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f2' changed: data type: 's1:{f1:string}' -> 's1:{f1:int}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f2' changed: data type: 's1:{f1:string}' -> 's1:{f1:int}'")); } @Test public void requireThatNestedDataTypeChangeInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type array<string> {} } field f2 type s1 { indexing: summary }", "struct s1 { field f1 type array<int> {} } field f2 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f2' changed: data type: 's1:{f1:Array<string>}' -> 's1:{f1:Array<int>}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f2' changed: data type: 's1:{f1:Array<string>}' -> 's1:{f1:Array<int>}'")); } @Test public void requireThatDataTypeChangeInNestedStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }", "struct s1 { field f1 type int {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f3' changed: data type: 's2:{s1:{f1:string}}' -> 's2:{s1:{f1:int}}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f3' changed: data type: 's2:{s1:{f1:string}}' -> 's2:{s1:{f1:int}}'")); } @Test public void requireThatMultipleDataTypeChangesInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} field f2 type int {} } field f3 type s1 { indexing: summary }", "struct s1 { field f1 type int {} field f2 type string {} } field f3 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f3' changed: data type: 's1:{f1:string,f2:int}' -> 's1:{f1:int,f2:string}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f3' changed: data type: 's1:{f1:string,f2:int}' -> 's1:{f1:int,f2:string}'")); } @Test @@ -197,7 +176,7 @@ public class DocumentTypeChangeValidatorTest { var validator = new DocumentTypeChangeValidator(ClusterSpec.Id.from("test"), createDocumentTypeWithReferenceField("oldDoc"), createDocumentTypeWithReferenceField("newDoc")); - List<VespaConfigChangeAction> result = validator.validate(ValidationOverrides.empty, Instant.now()); + List<VespaConfigChangeAction> result = validator.validate(); assertEquals(1, result.size()); VespaConfigChangeAction action = result.get(0); assertTrue(action instanceof VespaRefeedAction); @@ -210,21 +189,17 @@ public class DocumentTypeChangeValidatorTest { @Test public void changing_tensor_type_of_tensor_field_requires_refeed() throws Exception { + Instant.now(); new Fixture( "field f1 type tensor(x[2]) { indexing: attribute }", "field f1 type tensor(x[3]) { indexing: attribute }") - .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'tensor(x[2])' -> 'tensor(x[3])'", Instant.now())); + .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'tensor(x[2])' -> 'tensor(x[3])'")); + Instant.now(); new Fixture( "field f1 type tensor(x[5]) { indexing: attribute }", "field f1 type tensor(x[3]) { indexing: attribute }") - .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'tensor(x[5])' -> 'tensor(x[3])'", Instant.now())); + .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'tensor(x[5])' -> 'tensor(x[3])'")); } private static NewDocumentType createDocumentTypeWithReferenceField(String nameReferencedDocumentType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 9f418476a24..2e1ec53f886 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; @@ -56,12 +57,10 @@ public class IndexingScriptChangeValidatorTest { private static VespaConfigChangeAction expectedReindexingAction(String field, String changedMsg, String fromScript, String toScript) { return VespaReindexAction.of(ClusterSpec.Id.from("test"), - "indexing-change", - ValidationOverrides.empty, - "Field '" + field + "' changed: " + + ValidationId.indexingChange, + "Field '" + field + "' changed: " + (changedMsg.isEmpty() ? "" : changedMsg + ", ") + - "indexing script: '" + fromScript + "' -> '" + toScript + "'", - Instant.now()); + "indexing script: '" + fromScript + "' -> '" + toScript + "'"); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java index 04efffab438..0bc4ecbfdfd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java @@ -36,7 +36,7 @@ public class StructFieldAttributeChangeValidatorTestCase { public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(structFieldAttributeValidator.validate(ValidationOverrides.empty, Instant.now())); - result.addAll(docTypeValidator.validate(ValidationOverrides.empty, Instant.now())); + result.addAll(docTypeValidator.validate()); return result; } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java index 62a36422dd8..7d4be4b5e33 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @since 5.1.13 */ public class ContainerIncludeTest { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index 1ac95ac9a99..4993a51ab74 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -6,8 +6,10 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.http.AccessControl; +import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory; @@ -50,6 +52,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { FilterChains filterChains = http.getFilterChains(); assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID)); assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID)); + assertTrue(filterChains.hasChain(AccessControl.DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID)); } @Test @@ -297,6 +300,28 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { assertEquals(AccessControl.ClientAuthentication.want, http.getAccessControl().get().clientAuthentication); } + @Test + public void local_connector_has_default_chain() { + Http http = createModelAndGetHttp( + " <http>", + " <filtering>", + " <access-control/>", + " </filtering>", + " </http>"); + + Set<String> actualBindings = getFilterBindings(http, AccessControl.DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID); + assertThat(actualBindings, empty()); + + ConnectorFactory connectorFactory = http.getHttpServer().get().getConnectorFactories().stream() + .filter(cf -> cf.getListenPort() == Defaults.getDefaults().vespaWebServicePort()) + .findAny() + .get(); + + Optional<ComponentId> defaultChain = connectorFactory.getDefaultRequestFilterChain(); + assertTrue(defaultChain.isPresent()); + assertEquals(AccessControl.DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID, defaultChain.get()); + } + private Http createModelAndGetHttp(String... httpElement) { List<String> servicesXml = new ArrayList<>(); servicesXml.add("<container version='1.0'>"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 92e9ca43193..fe0b9841d1c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -993,29 +993,4 @@ public class ContentClusterTest extends ContentBaseTest { assertDirectStorageApiRpcFlagIsPropagatedToConfig(true); } - - @Test - public void use_fast_value_tensor_implementation_config_is_controlled_by_properties() { - assertUseFastValueTensorImplementationFlagIsPropagatedToConfig(false); - assertUseFastValueTensorImplementationFlagIsPropagatedToConfig(true); - } - - void assertUseFastValueTensorImplementationFlagIsPropagatedToConfig(boolean useFastValueTensorImplementation) { - VespaModel model = createEnd2EndOneNode(new TestProperties().setUseFastValueTensorImplementation(useFastValueTensorImplementation)); - ContentCluster cc = model.getContentClusters().get("storage"); - var node = cc.getSearch().getSearchNodes().get(0); - if (useFastValueTensorImplementation) { - assertTensorImplementationConfig(ProtonConfig.Tensor_implementation.FAST_VALUE, node); - } else { - assertTensorImplementationConfig(ProtonConfig.Tensor_implementation.TENSOR_ENGINE, node); - } - } - - void assertTensorImplementationConfig(ProtonConfig.Tensor_implementation.Enum exp, SearchNode node) { - var builder = new ProtonConfig.Builder(); - node.getConfig(builder); - var config = new ProtonConfig(builder); - assertEquals(exp, config.tensor_implementation()); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java index f1c86485a64..8f4c9f81d7f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java @@ -1,6 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import com.yahoo.config.Node; + import java.util.List; import java.util.Objects; @@ -53,6 +55,14 @@ public class ClusterResources { return true; } + /** Returns the total resources of this, that is the number of nodes times the node resources */ + public NodeResources totalResources() { + return nodeResources.withVcpu(nodeResources.vcpu() * nodes) + .withMemoryGb(nodeResources.memoryGb() * nodes) + .withDiskGb(nodeResources.diskGb() * nodes) + .withBandwidthGbps(nodeResources.bandwidthGbps() * nodes); + } + @Override public boolean equals(Object o) { if (o == this) return true; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java index cea40da52b9..f79abbddc56 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import ai.vespa.util.http.VespaHttpClientBuilder; @@ -72,23 +72,22 @@ public class ConfigVerification { for (Map.Entry<String, Stack<String>> entry : mappings.entrySet()) { recurseUrls.add(entry.getValue().pop()); } - int ret = compareOutputs(performRequests(recurseUrls, httpClient)); - if (ret != 0) { - return ret; - } + if ( ! equalOutputs(performRequests(recurseUrls, httpClient))) + return -1; } return 0; } - private static int compareOutputs(Map<String, String> outputs) { + private static boolean equalOutputs(Map<String, String> outputs) { Map.Entry<String, String> firstEntry = outputs.entrySet().iterator().next(); for (Map.Entry<String, String> entry : outputs.entrySet()) { if (!entry.getValue().equals(firstEntry.getValue())) { - System.out.println("output from '" + entry.getKey() + "' did not equal output from '" + firstEntry.getKey() + "'"); - return -1; + System.out.println("output from '" + entry.getKey() + "': '" + entry.getValue() + + "' did not equal output from '" + firstEntry.getKey() + "': '" + firstEntry.getValue() + "'"); + return false; } } - return 0; + return true; } private static String performRequest(String url, CloseableHttpClient httpClient) throws IOException { diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 33c568061fe..f22f5e5cb1c 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,6 +7,7 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true +tenants{}.applications{}.usePowerOfTwoChoicesLb bool default=false tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index 483e772b818..5aa0bb2ae9b 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -1,4 +1,4 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=cloud.config # Vespa home is prepended if the file is relative @@ -43,3 +43,5 @@ trustEmptySnapshot bool default=true tlsForQuorumCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" + +dynamicReconfiguration bool default=false diff --git a/configserver/pom.xml b/configserver/pom.xml index 8cd1b4b4254..35c38eb7d7d 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -34,6 +34,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <version>${project.version}</version> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index b356b99c50a..6cc05a0f69e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -326,12 +326,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Deployment deployment = prepare(sessionId, prepareParams, logger); - if (deployment.configChangeActions().getRefeedActions().getEntries().stream().anyMatch(entry -> ! entry.allowed())) - logger.log(Level.WARNING, "Activation rejected because of disallowed re-feed actions"); - else if (deployment.configChangeActions().getReindexActions().getEntries().stream().anyMatch(entry -> ! entry.allowed())) - logger.log(Level.WARNING, "Activation rejected because of disallowed re-index actions"); - else - deployment.activate(); + deployment.activate(); return new PrepareResult(sessionId, deployment.configChangeActions(), logger); } @@ -1014,21 +1009,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye RestartActions restartActions = actions.getRestartActions(); if ( ! restartActions.isEmpty()) { logger.log(Level.WARNING, "Change(s) between active and new application that require restart:\n" + - restartActions.format()); + restartActions.format()); } RefeedActions refeedActions = actions.getRefeedActions(); if ( ! refeedActions.isEmpty()) { - boolean allAllowed = refeedActions.getEntries().stream().allMatch(RefeedActions.Entry::allowed); - logger.log(allAllowed ? Level.INFO : Level.WARNING, + logger.log(Level.WARNING, "Change(s) between active and new application that may require re-feed:\n" + - refeedActions.format()); + refeedActions.format()); } ReindexActions reindexActions = actions.getReindexActions(); if ( ! reindexActions.isEmpty()) { - boolean allAllowed = reindexActions.getEntries().stream().allMatch(ReindexActions.Entry::allowed); - logger.log(allAllowed ? Level.INFO : Level.WARNING, - "Change(s) between active and new application that may require re-index:\n" + - reindexActions.format()); + logger.log(Level.WARNING, + "Change(s) between active and new application that may require re-index:\n" + + reindexActions.format()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index 2e73a02c75b..8d001d5d5df 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -130,7 +130,13 @@ public class Application implements ModelResult { if (logDebug()) { debug("Resolving " + configKey + " with config definition " + def); } - ConfigPayload payload = model.getConfig(configKey, def); + + ConfigPayload payload = null; + try { + payload = model.getConfig(configKey, def); + } catch (Exception e) { + throw new ConfigurationRuntimeException("Unable to get config for " + app, e); + } if (payload == null) { metricUpdater.incrementFailedRequests(); throw new ConfigurationRuntimeException("Unable to resolve config " + configKey); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java index ed9f12484b8..f98d58d9fb4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java @@ -73,11 +73,12 @@ public class ApplicationCuratorDatabase { /** * Creates a node for the given application, marking its existence. */ - public void createApplication(ApplicationId id) { + public void createApplication(ApplicationId id, Instant now) { if ( ! id.tenant().equals(tenant)) throw new IllegalArgumentException("Cannot write application id '" + id + "' for tenant '" + tenant + "'"); try (Lock lock = lock(id)) { curator.create(applicationPath(id)); + modifyReindexing(id, ApplicationReindexing.ready(now), UnaryOperator.identity()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ClusterReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ClusterReindexing.java new file mode 100644 index 00000000000..ca9aa01ea56 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ClusterReindexing.java @@ -0,0 +1,58 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Reindexing status for each document type in a content cluster. + * + * @author jonmv + */ +public class ClusterReindexing { + + private static final ClusterReindexing empty = new ClusterReindexing(Map.of()); + + private final Map<String, Status> documentTypeStatus; + + public ClusterReindexing(Map<String, Status> documentTypeStatus) { + this.documentTypeStatus = Map.copyOf(documentTypeStatus); + } + + public static ClusterReindexing empty() { return empty; } + + public Map<String, Status> documentTypeStatus() { return documentTypeStatus; } + + + public static class Status { + + private final Instant startedAt; + private final Instant endedAt; + private final State state; + private final String message; + private final String progress; + + public Status(Instant startedAt, Instant endedAt, State state, String message, String progress) { + this.startedAt = Objects.requireNonNull(startedAt); + this.endedAt = endedAt; + this.state = state; + this.message = message; + this.progress = progress; + } + + public Instant startedAt() { return startedAt; } + public Optional<Instant> endedAt() { return Optional.ofNullable(endedAt); } + public Optional<State> state() { return Optional.ofNullable(state); } + public Optional<String> message() { return Optional.ofNullable(message); } + public Optional<String> progress() { return Optional.ofNullable(progress); } + } + + + public enum State { + + PENDING, RUNNING, FAILED, SUCCESSFUL; + + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 6b316c06b54..434b254336c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -1,27 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import ai.vespa.util.http.VespaClientBuilderFactory; +import ai.vespa.util.http.VespaAsyncHttpClientBuilder; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.JSONResponse; -import org.glassfish.jersey.client.ClientProperties; -import org.glassfish.jersey.client.proxy.WebResourceFactory; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.ProcessingException; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientRequestFilter; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.HttpHeaders; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequests; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.core5.concurrent.FutureCallback; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.util.Timeout; + +import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URI; +import java.net.URISyntaxException; import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -29,8 +34,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; @@ -42,12 +54,12 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER * * @author Ulf Lilleengen * @author hmusum + * @author bjorncs */ public class ConfigConvergenceChecker extends AbstractComponent { private static final Logger log = Logger.getLogger(ConfigConvergenceChecker.class.getName()); - private static final String statePath = "/state/v1/"; - private static final String configSubPath = "config"; + private final static Set<String> serviceTypesToCheck = Set.of( CONTAINER.serviceName, QRSERVER.serviceName, @@ -58,17 +70,13 @@ public class ConfigConvergenceChecker extends AbstractComponent { "distributor" ); - private final StateApiFactory stateApiFactory; - private final VespaClientBuilderFactory clientBuilderFactory = new VespaClientBuilderFactory(); - @Inject - public ConfigConvergenceChecker() { - this(ConfigConvergenceChecker::createStateApi); - } + private final Executor responseHandlerExecutor = + Executors.newSingleThreadExecutor(new DaemonThreadFactory("config-convergence-checker-response-handler-")); + private final ObjectMapper jsonMapper = new ObjectMapper(); - public ConfigConvergenceChecker(StateApiFactory stateApiFactory) { - this.stateApiFactory = stateApiFactory; - } + @Inject + public ConfigConvergenceChecker() {} /** Fetches the active config generation for all services in the given application. */ public Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService) { @@ -82,7 +90,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { } /** Check all services in given application. Returns the minimum current generation of all services */ - public ServiceListResponse getServiceConfigGenerationsResponse(Application application, URI requestUrl, Duration timeoutPerService) { + public JSONResponse getServiceConfigGenerationsResponse(Application application, URI requestUrl, Duration timeoutPerService) { Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService); long currentGeneration = currentGenerations.values().stream().mapToLong(Long::longValue).min().orElse(-1); return new ServiceListResponse(200, currentGenerations, requestUrl, application.getApplicationGeneration(), @@ -90,64 +98,81 @@ public class ConfigConvergenceChecker extends AbstractComponent { } /** Check service identified by host and port in given application */ - public ServiceResponse getServiceConfigGenerationResponse(Application application, String hostAndPortToCheck, URI requestUrl, Duration timeout) { + public JSONResponse getServiceConfigGenerationResponse(Application application, String hostAndPortToCheck, URI requestUrl, Duration timeout) { Long wantedGeneration = application.getApplicationGeneration(); - try { + try (CloseableHttpAsyncClient client = createHttpClient()) { + client.start(); if ( ! hostInApplication(application, hostAndPortToCheck)) return ServiceResponse.createHostNotFoundInAppResponse(requestUrl, hostAndPortToCheck, wantedGeneration); - - long currentGeneration = getServiceGeneration(URI.create("http://" + hostAndPortToCheck), timeout); + long currentGeneration = getServiceGeneration(client, URI.create("http://" + hostAndPortToCheck), timeout).get(); boolean converged = currentGeneration >= wantedGeneration; return ServiceResponse.createOkResponse(requestUrl, hostAndPortToCheck, wantedGeneration, currentGeneration, converged); - } catch (ProcessingException e) { // e.g. if we cannot connect to the service to find generation + } catch (InterruptedException | ExecutionException | CancellationException e) { // e.g. if we cannot connect to the service to find generation return ServiceResponse.createNotFoundResponse(requestUrl, hostAndPortToCheck, wantedGeneration, e.getMessage()); } catch (Exception e) { return ServiceResponse.createErrorResponse(requestUrl, hostAndPortToCheck, wantedGeneration, e.getMessage()); } } - @Override - public void deconstruct() { - clientBuilderFactory.close(); - } - - @Path(statePath) - public interface StateApi { - @Path(configSubPath) - @GET - JsonNode config(); - } - - public interface StateApiFactory { - StateApi createStateApi(Client client, URI serviceUri); - } - /** Gets service generation for a list of services (in parallel). */ private Map<ServiceInfo, Long> getServiceGenerations(List<ServiceInfo> services, Duration timeout) { - return services.parallelStream() - .collect(Collectors.toMap(service -> service, - service -> { - try { - return getServiceGeneration(URI.create("http://" + service.getHostName() - + ":" + getStatePort(service).get()), timeout); - } - catch (ProcessingException e) { // Cannot connect to service to determine service generation - return -1L; - } - }, - (v1, v2) -> { throw new IllegalStateException("Duplicate keys for values '" + v1 + "' and '" + v2 + "'."); }, - LinkedHashMap::new - )); + try (CloseableHttpAsyncClient client = createHttpClient()) { + client.start(); + List<CompletableFuture<Void>> inprogressRequests = new ArrayList<>(); + ConcurrentMap<ServiceInfo, Long> temporaryResult = new ConcurrentHashMap<>(); + for (ServiceInfo service : services) { + int statePort = getStatePort(service).orElse(0); + if (statePort <= 0) continue; + URI uri = URI.create("http://" + service.getHostName() + ":" + statePort); + CompletableFuture<Void> inprogressRequest = getServiceGeneration(client, uri, timeout) + .handle((result, error) -> { + if (result != null) { + temporaryResult.put(service, result); + } else { + log.log( + Level.FINE, + error, + () -> String.format("Failed to retrieve service config generation for '%s': %s", service, error.getMessage())); + temporaryResult.put(service, -1L); + } + return null; + }); + inprogressRequests.add(inprogressRequest); + } + CompletableFuture.allOf(inprogressRequests.toArray(CompletableFuture[]::new)).join(); + return createMapOrderedByServiceList(services, temporaryResult); + } catch (IOException e) { + // Actual client implementation does not throw IOException on close() + throw new UncheckedIOException(e); + } } /** Get service generation of service at given URL */ - private long getServiceGeneration(URI serviceUrl, Duration timeout) { - Client client = createClient(timeout); + private CompletableFuture<Long> getServiceGeneration(CloseableHttpAsyncClient client, URI serviceUrl, Duration timeout) { + SimpleHttpRequest request = SimpleHttpRequests.get(createApiUri(serviceUrl)); + request.setHeader("Connection", "close"); + request.setConfig(createRequestConfig(timeout)); + + // Ignoring returned Future object as we want to use the more flexible CompletableFuture instead + CompletableFuture<SimpleHttpResponse> responsePromise = new CompletableFuture<>(); + client.execute(request, new FutureCallback<>() { + @Override public void completed(SimpleHttpResponse result) { responsePromise.complete(result); } + @Override public void failed(Exception ex) { responsePromise.completeExceptionally(ex); } + @Override public void cancelled() { responsePromise.cancel(false); } + }); + + // Don't do json parsing in http client's thread + return responsePromise.thenApplyAsync(this::handleResponse, responseHandlerExecutor); + } + + private long handleResponse(SimpleHttpResponse response) throws UncheckedIOException { try { - StateApi state = stateApiFactory.createStateApi(client, serviceUrl); - return generationFromContainerState(state.config()); - } finally { - client.close(); + int statusCode = response.getCode(); + if (statusCode != HttpStatus.SC_OK) throw new IOException("Expected status code 200, got " + statusCode); + if (response.getBody() == null) throw new IOException("Response has no content"); + return generationFromContainerState(jsonMapper.readTree(response.getBodyText())); + } catch (IOException e) { + throw new UncheckedIOException(e); } } @@ -166,16 +191,6 @@ public class ConfigConvergenceChecker extends AbstractComponent { return false; } - private Client createClient(Duration timeout) { - return clientBuilderFactory.newBuilder() - .register( - (ClientRequestFilter) ctx -> - ctx.getHeaders().put(HttpHeaders.USER_AGENT, List.of("config-convergence-checker"))) - .property(ClientProperties.CONNECT_TIMEOUT, (int) timeout.toMillis()) - .property(ClientProperties.READ_TIMEOUT, (int) timeout.toMillis()) - .build(); - } - private static Optional<Integer> getStatePort(ServiceInfo service) { return service.getPorts().stream() .filter(port -> port.getTags().contains("state")) @@ -187,9 +202,47 @@ public class ConfigConvergenceChecker extends AbstractComponent { return state.get("config").get("generation").asLong(-1); } - private static StateApi createStateApi(Client client, URI uri) { - WebTarget target = client.target(uri); - return WebResourceFactory.newResource(StateApi.class, target); + private static Map<ServiceInfo, Long> createMapOrderedByServiceList( + List<ServiceInfo> services, ConcurrentMap<ServiceInfo, Long> result) { + Map<ServiceInfo, Long> orderedResult = new LinkedHashMap<>(); + for (ServiceInfo service : services) { + Long generation = result.get(service); + if (generation != null) { + orderedResult.put(service, generation); + } + } + return orderedResult; + } + + private static URI createApiUri(URI serviceUrl) { + try { + return new URIBuilder(serviceUrl) + .setPath("/state/v1/config") + .build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + + private static RequestConfig createRequestConfig(Duration timeout) { + return RequestConfig.custom() + .setConnectionRequestTimeout(Timeout.ofSeconds(1)) + .setResponseTimeout(Timeout.ofMilliseconds(timeout.toMillis())) + .setConnectTimeout(Timeout.ofSeconds(1)) + .build(); + } + + private static CloseableHttpAsyncClient createHttpClient() { + return VespaAsyncHttpClientBuilder + .create(tlsStrategy -> + PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnTotal(100) + .setMaxConnPerRoute(10) + .setTlsStrategy(tlsStrategy) + .build()) + .setUserAgent("config-convergence-checker") + .setConnectionReuseStrategy((request, response, context) -> false) // Disable connection reuse + .build(); } private static class ServiceListResponse extends JSONResponse { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index f07a595a830..0eeb9759a39 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -64,13 +64,13 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica private final HostRegistry<ApplicationId> hostRegistry; private final ApplicationMapper applicationMapper = new ApplicationMapper(); private final MetricUpdater tenantMetricUpdater; - private final Clock clock = Clock.systemUTC(); + private final Clock clock; private final TenantFileSystemDirs tenantFileSystemDirs; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, ExecutorService zkCacheExecutor, Metrics metrics, ReloadListener reloadListener, ConfigserverConfig configserverConfig, HostRegistry<ApplicationId> hostRegistry, - TenantFileSystemDirs tenantFileSystemDirs) { + TenantFileSystemDirs tenantFileSystemDirs, Clock clock) { this.database = new ApplicationCuratorDatabase(tenant, curator); this.tenant = tenant; this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command); @@ -83,6 +83,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); this.hostRegistry = hostRegistry; this.tenantFileSystemDirs = tenantFileSystemDirs; + this.clock = clock; } // For testing only @@ -95,7 +96,8 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica componentRegistry.getReloadListener(), componentRegistry.getConfigserverConfig(), componentRegistry.getHostRegistries().createApplicationHostRegistry(tenantName), - new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName)); + new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName), + componentRegistry.getClock()); } /** The curator backed ZK storage of this. */ @@ -140,7 +142,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica * Creates a node for the given application, marking its existence. */ public void createApplication(ApplicationId id) { - database().createApplication(id); + database().createApplication(id, clock.instant()); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java index ec48b671a5b..1a0d109b6c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java @@ -42,7 +42,6 @@ public class ConfigChangeActionsSlimeConverter { for (RefeedActions.Entry entry : actions.getRefeedActions().getEntries()) { Cursor entryCursor = refeedCursor.addObject(); entryCursor.setString("name", entry.name()); - entryCursor.setBool("allowed", entry.allowed()); entryCursor.setString("documentType", entry.getDocumentType()); entryCursor.setString("clusterName", entry.getClusterName()); messagesToSlime(entryCursor, entry.getMessages()); @@ -55,7 +54,6 @@ public class ConfigChangeActionsSlimeConverter { for (ReindexActions.Entry entry : actions.getReindexActions().getEntries()) { Cursor entryCursor = refeedCursor.addObject(); entryCursor.setString("name", entry.name()); - entryCursor.setBool("allowed", entry.allowed()); entryCursor.setString("documentType", entry.getDocumentType()); entryCursor.setString("clusterName", entry.getClusterName()); messagesToSlime(entryCursor, entry.getMessages()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java index c20b8527f2e..b2221cbcf6c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java @@ -5,7 +5,13 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; -import java.util.*; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; /** * Represents all actions to re-feed document types in order to handle config changes. @@ -17,15 +23,13 @@ public class RefeedActions { public static class Entry { private final String name; - private final boolean allowed; private final String documentType; private final String clusterName; private final Set<ServiceInfo> services = new LinkedHashSet<>(); private final Set<String> messages = new TreeSet<>(); - private Entry(String name, boolean allowed, String documentType, String clusterName) { + private Entry(String name, String documentType, String clusterName) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; } @@ -42,8 +46,6 @@ public class RefeedActions { public String name() { return name; } - public boolean allowed() { return allowed; } - public String getDocumentType() { return documentType; } public String getClusterName() { return clusterName; } @@ -54,12 +56,12 @@ public class RefeedActions { } - private Entry addEntry(String name, boolean allowed, String documentType, ServiceInfo service) { + private Entry addEntry(String name, String documentType, ServiceInfo service) { String clusterName = service.getProperty("clustername").orElse(""); - String entryId = name + "." + allowed + "." + clusterName + "." + documentType; + String entryId = name + "." + "." + clusterName + "." + documentType; Entry entry = actions.get(entryId); if (entry == null) { - entry = new Entry(name, allowed, documentType, clusterName); + entry = new Entry(name, documentType, clusterName); actions.put(entryId, entry); } return entry; @@ -75,7 +77,7 @@ public class RefeedActions { if (action.getType().equals(ConfigChangeAction.Type.REFEED)) { ConfigChangeRefeedAction refeedAction = (ConfigChangeRefeedAction) action; for (ServiceInfo service : refeedAction.getServices()) { - addEntry(refeedAction.name(), refeedAction.allowed(), refeedAction.getDocumentType(), service). + addEntry(refeedAction.name(), refeedAction.getDocumentType(), service). addService(service). addMessage(action.getMessage()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java index 425276cebd6..6e2e23ab6be 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java @@ -18,8 +18,6 @@ public class RefeedActionsFormatter { public String format() { StringBuilder builder = new StringBuilder(); for (RefeedActions.Entry entry : actions.getEntries()) { - if (entry.allowed()) - builder.append("(allowed) "); builder.append(entry.name() + ": Consider removing data and re-feed document type '" + entry.getDocumentType() + "' in cluster '" + entry.getClusterName() + "' because:\n"); int counter = 1; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java index e328f9595b7..6ed1c43623f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java @@ -27,7 +27,7 @@ public class ReindexActions { if (action.getType().equals(ConfigChangeAction.Type.REINDEX)) { ConfigChangeReindexAction reindexChange = (ConfigChangeReindexAction) action; for (ServiceInfo service : reindexChange.getServices()) { - addEntry(reindexChange.name(), reindexChange.allowed(), reindexChange.getDocumentType(), service). + addEntry(reindexChange.name(), reindexChange.getDocumentType(), service). addService(service). addMessage(action.getMessage()); } @@ -35,12 +35,12 @@ public class ReindexActions { } } - private Entry addEntry(String name, boolean allowed, String documentType, ServiceInfo service) { + private Entry addEntry(String name, String documentType, ServiceInfo service) { String clusterName = service.getProperty("clustername").orElse(""); - String entryId = name + "." + allowed + "." + clusterName + "." + documentType; + String entryId = name + "." + "." + clusterName + "." + documentType; Entry entry = actions.get(entryId); if (entry == null) { - entry = new Entry(name, allowed, documentType, clusterName); + entry = new Entry(name, documentType, clusterName); actions.put(entryId, entry); } return entry; @@ -53,15 +53,13 @@ public class ReindexActions { public static class Entry { private final String name; - private final boolean allowed; private final String documentType; private final String clusterName; private final Set<ServiceInfo> services = new LinkedHashSet<>(); private final Set<String> messages = new TreeSet<>(); - private Entry(String name, boolean allowed, String documentType, String clusterName) { + private Entry(String name, String documentType, String clusterName) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; } @@ -77,7 +75,6 @@ public class ReindexActions { } public String name() { return name; } - public boolean allowed() { return allowed; } public String getDocumentType() { return documentType; } public String getClusterName() { return clusterName; } public Set<ServiceInfo> getServices() { return services; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java index e89bfd522cd..bdd01404f64 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java @@ -17,8 +17,6 @@ class ReindexActionsFormatter { String format() { StringBuilder builder = new StringBuilder(); for (ReindexActions.Entry entry : actions.getEntries()) { - if (entry.allowed()) - builder.append("(allowed) "); builder.append(entry.name() + ": Consider re-indexing document type '" + entry.getDocumentType() + "' in cluster '" + entry.getClusterName() + "' because:\n"); int counter = 1; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 6fb315dc3b3..b924e07ff47 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -18,6 +18,7 @@ import com.yahoo.jdisc.application.UriPattern; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.application.ApplicationReindexing; +import com.yahoo.vespa.config.server.application.ClusterReindexing; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -29,7 +30,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Stream; @@ -249,9 +252,10 @@ public class ApplicationHandler extends HttpHandler { if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant().value() + "' not found"); - return new ReindexResponse(tenant.getApplicationRepo().database() - .readReindexingStatus(applicationId) - .orElseThrow(() -> new NotFoundException("Reindexing status not found for " + applicationId))); + return new ReindexingResponse(tenant.getApplicationRepo().database() + .readReindexingStatus(applicationId) + .orElseThrow(() -> new NotFoundException("Reindexing status not found for " + applicationId)), + Map.of()); // TODO jonmv/bjorncs: Get status of each cluster and fill in here. } private HttpResponse restart(HttpRequest request, ApplicationId applicationId) { @@ -439,31 +443,59 @@ public class ApplicationHandler extends HttpHandler { } } - private static class ReindexResponse extends JSONResponse { - ReindexResponse(ApplicationReindexing reindexing) { + static class ReindexingResponse extends JSONResponse { + ReindexingResponse(ApplicationReindexing reindexing, Map<String, ClusterReindexing> clusters) { super(Response.Status.OK); - object.setBool("enabled", reindexing.enabled()); - setStatus(object.setObject("status"), reindexing.common()); - - Cursor clustersObject = object.setObject("clusters"); - reindexing.clusters().entrySet().stream().sorted(comparingByKey()) - .forEach(cluster -> { - Cursor clusterObject = clustersObject.setObject(cluster.getKey()); - setStatus(clusterObject.setObject("status"), cluster.getValue().common()); - - Cursor pendingObject = clusterObject.setObject("pending"); - cluster.getValue().pending().entrySet().stream().sorted(comparingByKey()) - .forEach(pending -> pendingObject.setLong(pending.getKey(), pending.getValue())); - - Cursor readyObject = clusterObject.setObject("ready"); - cluster.getValue().ready().entrySet().stream().sorted(comparingByKey()) - .forEach(ready -> setStatus(readyObject.setObject(ready.getKey()), ready.getValue())); - }); + object.setBool("enabled", reindexing.enabled()); + setStatus(object.setObject("status"), reindexing.common()); + + Cursor clustersObject = object.setObject("clusters"); + Stream<String> clusterNames = Stream.concat(clusters.keySet().stream(), reindexing.clusters().keySet().stream()); + clusterNames.sorted() + .forEach(clusterName -> { + Cursor clusterObject = clustersObject.setObject(clusterName); + Cursor pendingObject = clusterObject.setObject("pending"); + Cursor readyObject = clusterObject.setObject("ready"); + + Map<String, Cursor> statuses = new HashMap<>(); + if (reindexing.clusters().containsKey(clusterName)) { + setStatus(clusterObject.setObject("status"), reindexing.clusters().get(clusterName).common()); + + reindexing.clusters().get(clusterName).pending().entrySet().stream().sorted(comparingByKey()) + .forEach(pending -> pendingObject.setLong(pending.getKey(), pending.getValue())); + + reindexing.clusters().get(clusterName).ready().entrySet().stream().sorted(comparingByKey()) + .forEach(ready -> setStatus(statuses.computeIfAbsent(ready.getKey(), readyObject::setObject), ready.getValue())); + } + if (clusters.containsKey(clusterName)) + clusters.get(clusterName).documentTypeStatus().entrySet().stream().sorted(comparingByKey()) + .forEach(status -> setStatus(statuses.computeIfAbsent(status.getKey(), readyObject::setObject), status.getValue())); + + }); } - private static void setStatus(Cursor object, ApplicationReindexing.Status status) { - object.setLong("readyMillis", status.ready().toEpochMilli()); + private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) { + object.setLong("readyMillis", readyStatus.ready().toEpochMilli()); } + + private static void setStatus(Cursor object, ClusterReindexing.Status status) { + object.setLong("startedMillis", status.startedAt().toEpochMilli()); + status.endedAt().ifPresent(endedAt -> object.setLong("endedMillis", endedAt.toEpochMilli())); + status.state().map(ReindexingResponse::toString).ifPresent(state -> object.setString("state", state)); + status.message().ifPresent(message -> object.setString("message", message)); + status.progress().ifPresent(progress -> object.setString("progress", progress)); + } + + static String toString(ClusterReindexing.State state) { + switch (state) { + case PENDING: return "pending"; + case RUNNING: return "running"; + case FAILED: return "failed"; + case SUCCESSFUL: return "successful"; + default: throw new IllegalArgumentException("Unexpected state '" + state + "'"); + } + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index 19534bba810..e59c334b89f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -1,12 +1,12 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.maintenance; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import java.time.Duration; +import java.util.logging.Level; /** * Removes expired sessions and locks @@ -17,6 +17,7 @@ import java.time.Duration; */ public class SessionsMaintainer extends ConfigServerMaintainer { private final boolean hostedVespa; + private int iteration = 0; SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FlagSource flagSource) { super(applicationRepository, curator, flagSource, Duration.ofMinutes(1), interval); @@ -25,14 +26,18 @@ public class SessionsMaintainer extends ConfigServerMaintainer { @Override protected boolean maintain() { + if (iteration % 10 == 0) + log.log(Level.INFO, () -> "Running " + SessionsMaintainer.class.getSimpleName() + ", iteration " + iteration); + applicationRepository.deleteExpiredLocalSessions(); if (hostedVespa) { Duration expiryTime = Duration.ofMinutes(90); int deleted = applicationRepository.deleteExpiredRemoteSessions(expiryTime); - log.log(LogLevel.FINE, () -> "Deleted " + deleted + " expired remote sessions older than " + expiryTime); + log.log(Level.FINE, () -> "Deleted " + deleted + " expired remote sessions older than " + expiryTime); } + iteration++; return true; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index ae258445e88..d816c3215a7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,7 +9,10 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -32,10 +35,12 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; + private final BooleanFlag usePowerOfTwoChoicesLb; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; + usePowerOfTwoChoicesLb = Flags.USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING.bindTo(flagSource); } @Override @@ -67,6 +72,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); + ab.usePowerOfTwoChoicesLb(usePowerOfTwoChoicesLb(app)); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); @@ -87,6 +93,10 @@ public class LbServicesProducer implements LbServicesConfig.Producer { return activeRotation; } + private boolean usePowerOfTwoChoicesLb(ApplicationInfo app) { + return usePowerOfTwoChoicesLb.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value(); + } + private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) { LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder(); hb.hostname(hostInfo.getHostname()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 5ddad540d8e..4e3fae37d63 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -242,7 +242,8 @@ public class TenantRepository { componentRegistry.getReloadListener(), componentRegistry.getConfigserverConfig(), componentRegistry.getHostRegistries().createApplicationHostRegistry(tenantName), - new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName)); + new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName), + componentRegistry.getClock()); SessionRepository sessionRepository = new SessionRepository(tenantName, componentRegistry, applicationRepo, diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ConfigServerLocation.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ConfigServerLocation.java index cc452421d2d..05d1119aa4f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/ConfigServerLocation.java +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/ConfigServerLocation.java @@ -15,6 +15,7 @@ public class ConfigServerLocation extends AbstractComponent { final int restApiPort; // The client factory must be owned by a component as StateResource is instantiated per request + @SuppressWarnings("removal") final VespaClientBuilderFactory clientBuilderFactory = new VespaClientBuilderFactory(); @Inject diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateResource.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateResource.java index 76e600d2ad8..138e6c8798c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateResource.java +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateResource.java @@ -40,6 +40,7 @@ public class StateResource implements StateClient { private static final String USER_AGENT = "service-view-config-server-client"; private static final String SINGLE_API_LINK = "url"; + @SuppressWarnings("removal") private final VespaClientBuilderFactory clientBuilderFactory; private final int restApiPort; private final String host; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 6aeb774d2b0..8a3a3b6e0ca 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -2,15 +2,13 @@ package com.yahoo.vespa.config.server.application; import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.yahoo.component.Version; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import org.junit.Before; @@ -31,8 +29,9 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.yahoo.test.json.JsonTestHelper.assertJsonEquals; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -184,10 +183,15 @@ public class ConfigConvergenceCheckerTest { .withBody("response too slow"))); HttpResponse response = checker.getServiceConfigGenerationResponse(application, hostAndPort(service), requestUrl, Duration.ofMillis(1)); // Message contained in a SocketTimeoutException may differ across platforms, so we do a partial match of the response here - assertResponse((responseBody) -> assertTrue("Response matches", responseBody.startsWith( - "{\"url\":\"" + requestUrl.toString() + "\",\"host\":\"" + hostAndPort(requestUrl) + - "\",\"wantedGeneration\":3,\"error\":\"java.net.SocketTimeoutException") && - responseBody.endsWith("\"}")), 404, response); + assertResponse( + responseBody -> + assertThat(responseBody) + .startsWith("{\"url\":\"" + requestUrl.toString() + "\",\"host\":\"" + hostAndPort(requestUrl) + + "\",\"wantedGeneration\":3,\"error\":\"") + .contains("java.net.SocketTimeoutException: 1 MILLISECONDS") + .endsWith("\"}"), + 404, + response); } private URI testServer() { @@ -202,16 +206,8 @@ public class ConfigConvergenceCheckerTest { return uri.getHost() + ":" + uri.getPort(); } - private static void assertResponse(String json, int status, HttpResponse response) { - assertResponse((responseBody) -> { - Slime expected = SlimeUtils.jsonToSlime(json.getBytes()); - Slime actual = SlimeUtils.jsonToSlime(responseBody.getBytes()); - try { - assertEquals(new String((SlimeUtils.toJsonBytes(expected))), new String(SlimeUtils.toJsonBytes(actual))); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }, status, response); + private static void assertResponse(String expectedJson, int status, HttpResponse response) { + assertResponse((responseBody) -> assertJsonEquals(new String(responseBody.getBytes()), expectedJson), status, response); } private static void assertResponse(Consumer<String> assertFunc, int status, HttpResponse response) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index b5194432682..876b169742f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -16,7 +16,6 @@ import java.util.List; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * @author geirst @@ -44,20 +43,16 @@ public class ConfigChangeActionsBuilder { } - ConfigChangeActionsBuilder refeed(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { - actions.add(new MockRefeedAction(name, - allowed, + ConfigChangeActionsBuilder refeed(ValidationId validationId, String message, String documentType, String clusterName, String serviceName) { + actions.add(new MockRefeedAction(validationId, message, List.of(createService(clusterName, "myclustertype", "myservicetype", serviceName)), documentType)); return this; } - ConfigChangeActionsBuilder reindex(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { + ConfigChangeActionsBuilder reindex(ValidationId validationId, String message, String documentType, String clusterName, String serviceName) { List<ServiceInfo> services = List.of(createService(clusterName, "myclustertype", "myservicetype", serviceName)); - ValidationOverrides overrides = mock(ValidationOverrides.class); - when(overrides.allows((String) any(), any())).thenReturn(allowed); - when(overrides.allows((ValidationId) any(), any())).thenReturn(allowed); - actions.add(VespaReindexAction.of(ClusterSpec.Id.from(clusterName), name, overrides, message, services, documentType, Instant.now())); + actions.add(VespaReindexAction.of(ClusterSpec.Id.from(clusterName), validationId, message, services, documentType)); return this; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java index d145a796725..d75f95d4c48 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java @@ -89,16 +89,15 @@ public class ConfigChangeActionsSlimeConverterTest { @Test public void json_representation_of_refeed_actions() throws IOException { ConfigChangeActions actions = new ConfigChangeActionsBuilder(). - refeed(CHANGE_ID, true, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE). - refeed(CHANGE_ID_2, false, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_TYPE).build(); + refeed(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE). + refeed(CHANGE_ID_2, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_TYPE).build(); assertEquals("{\n" + " \"configChangeActions\": {\n" + " \"restart\": [\n" + " ],\n" + " \"refeed\": [\n" + " {\n" + - " \"name\": \"change-id\",\n" + - " \"allowed\": true,\n" + + " \"name\": \"field-type-change\",\n" + " \"documentType\": \"music\",\n" + " \"clusterName\": \"foo\",\n" + " \"messages\": [\n" + @@ -114,8 +113,7 @@ public class ConfigChangeActionsSlimeConverterTest { " ]\n" + " },\n" + " {\n" + - " \"name\": \"other-change-id\",\n" + - " \"allowed\": false,\n" + + " \"name\": \"indexing-change\",\n" + " \"documentType\": \"book\",\n" + " \"clusterName\": \"foo\",\n" + " \"messages\": [\n" + @@ -141,7 +139,7 @@ public class ConfigChangeActionsSlimeConverterTest { @Test public void json_representation_of_reindex_actions() throws IOException { ConfigChangeActions actions = new ConfigChangeActionsBuilder(). - reindex(CHANGE_ID, true, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE).build(); + reindex(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE).build(); assertEquals( "{\n" + " \"configChangeActions\": {\n" + @@ -151,8 +149,7 @@ public class ConfigChangeActionsSlimeConverterTest { " ],\n" + " \"reindex\": [\n" + " {\n" + - " \"name\": \"change-id\",\n" + - " \"allowed\": true,\n" + + " \"name\": \"field-type-change\",\n" + " \"documentType\": \"music\",\n" + " \"clusterName\": \"foo\",\n" + " \"messages\": [\n" + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java index 11f2a46994c..615d4c86c1d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java @@ -1,32 +1,35 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import java.util.List; +import java.util.Optional; /** * @author geirst */ public class MockRefeedAction extends MockConfigChangeAction implements ConfigChangeRefeedAction { - private final String name; - private final boolean allowed; + private final ValidationId validationId; private final String documentType; - public MockRefeedAction(String name, boolean allowed, String message, List<ServiceInfo> services, String documentType) { + public MockRefeedAction(ValidationId validationId, String message, List<ServiceInfo> services, String documentType) { super(message, services); - this.name = name; - this.allowed = allowed; + this.validationId = validationId; this.documentType = documentType; } @Override - public String name() { return name; } + public Optional<ValidationId> validationId() { return Optional.of(validationId); } @Override - public boolean allowed() { return allowed; } + public ClusterSpec.Id clusterId() { + return null; + } @Override public boolean ignoreForInternalRedeploy() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java index 48d6833129e..4b898b501ec 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java @@ -15,9 +15,9 @@ public class RefeedActionsFormatterTest { @Test public void formatting_of_single_action() { RefeedActions actions = new ConfigChangeActionsBuilder(). - refeed(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). build().getRefeedActions(); - assertEquals("change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + " 1) change\n", new RefeedActionsFormatter(actions).format()); } @@ -25,20 +25,18 @@ public class RefeedActionsFormatterTest { @Test public void formatting_of_multiple_actions() { RefeedActions actions = new ConfigChangeActionsBuilder(). - refeed(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID_2, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID_2, true, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). build().getRefeedActions(); - assertEquals("change-id: Consider removing data and re-feed document type 'book' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider removing data and re-feed document type 'book' in cluster 'foo' because:\n" + " 1) other change\n" + - "change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + "field-type-change: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + " 1) change\n" + " 2) other change\n" + - "other-change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + - " 1) other change\n" + - "(allowed) other-change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + "indexing-change: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + " 1) other change\n", new RefeedActionsFormatter(actions).format()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java index 7235b8905c5..24e81dc3f99 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ServiceInfo; import org.junit.Test; @@ -32,8 +33,8 @@ public class RefeedActionsTest { @Test public void action_with_multiple_reasons() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(1)); assertThat(toString(entries.get(0)), equalTo("music.foo:[baz][change,other change]")); @@ -42,8 +43,8 @@ public class RefeedActionsTest { @Test public void actions_with_multiple_services() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME_2). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME_2). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(1)); assertThat(toString(entries.get(0)), equalTo("music.foo:[baz,qux][change]")); @@ -52,8 +53,8 @@ public class RefeedActionsTest { @Test public void actions_with_multiple_document_types() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_NAME). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(2)); assertThat(toString(entries.get(0)), equalTo("book.foo:[baz][change]")); @@ -63,8 +64,8 @@ public class RefeedActionsTest { @Test public void actions_with_multiple_clusters() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER_2, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER_2, SERVICE_NAME). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(2)); assertThat(toString(entries.get(0)), equalTo("music.bar:[baz][change]")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java index e9dd3f3bbfc..b07d002a431 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java @@ -21,9 +21,9 @@ public class ReindexActionsFormatterTest { @Test public void formatting_of_single_action() { ReindexActions actions = new ConfigChangeActionsBuilder(). - reindex(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). build().getReindexActions(); - assertEquals("change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + " 1) change\n", new ReindexActionsFormatter(actions).format()); } @@ -31,20 +31,18 @@ public class ReindexActionsFormatterTest { @Test public void formatting_of_multiple_actions() { ReindexActions actions = new ConfigChangeActionsBuilder(). - reindex(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID_2, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID_2, true, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). build().getReindexActions(); - assertEquals("change-id: Consider re-indexing document type 'book' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider re-indexing document type 'book' in cluster 'foo' because:\n" + " 1) other change\n" + - "change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + + "field-type-change: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + " 1) change\n" + " 2) other change\n" + - "other-change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + - " 1) other change\n" + - "(allowed) other-change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + + "indexing-change: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + " 1) other change\n", new ReindexActionsFormatter(actions).format()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java index 8499c12f648..e02e1e2b143 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java @@ -1,14 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; +import com.yahoo.config.application.api.ValidationId; + /** * @author geirst * @since 5.44 */ public class Utils { - final static String CHANGE_ID = "change-id"; - final static String CHANGE_ID_2 = "other-change-id"; + final static ValidationId CHANGE_ID = ValidationId.fieldTypeChange; + final static ValidationId CHANGE_ID_2 = ValidationId.indexingChange; final static String CHANGE_MSG = "change"; final static String CHANGE_MSG_2 = "other change"; final static String DOC_TYPE = "music"; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 4d1b9341e7f..341fa7109da 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; -import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; @@ -49,6 +49,7 @@ import static com.yahoo.vespa.config.server.deploy.DeployTester.createFailingMod import static com.yahoo.vespa.config.server.deploy.DeployTester.createHostedModelFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -390,27 +391,6 @@ public class HostedDeployTest { } @Test - public void testThatDisallowedConfigChangeActionsBlockDeployment() throws IOException { - List<Host> hosts = List.of(createHost("host1", "6.1.0"), - createHost("host2", "6.1.0"), - createHost("host3", "6.1.0"), - createHost("host4", "6.1.0")); - List<ServiceInfo> services = List.of( - new ServiceInfo("serviceName", "serviceType", null, Map.of("clustername", "cluster"), "configId", "hostName")); - - ManualClock clock = new ManualClock(Instant.EPOCH); - List<ModelFactory> modelFactories = List.of( - new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), - VespaReindexAction.of(ClusterSpec.Id.from("test"), "indexing-mode-change", ValidationOverrides.empty, - "reindex please", services, "music", clock.instant()), - new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services))); - - DeployTester tester = createTester(hosts, modelFactories, prodZone, clock); - PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.1.0"); - assertNull("Deployment was not activated", tester.applicationRepository().getActiveSession(tester.applicationId())); - } - - @Test public void testThatAllowedConfigChangeActionsAreActedUpon() throws IOException { List<Host> hosts = List.of(createHost("host1", "6.1.0"), createHost("host2", "6.1.0"), @@ -422,8 +402,8 @@ public class HostedDeployTest { ManualClock clock = new ManualClock(Instant.EPOCH); List<ModelFactory> modelFactories = List.of( new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), - VespaReindexAction.of(ClusterSpec.Id.from("test"), "indexing-mode-change", ValidationOverrides.all, - "reindex please", services, "music", clock.instant()), + VespaReindexAction.of(ClusterSpec.Id.from("test"), ValidationId.indexModeChange, + "reindex please", services, "music"), new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services))); DeployTester tester = createTester(hosts, modelFactories, prodZone, clock); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index a34de472d1e..910c4b069e3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.model.api.ModelFactory; @@ -15,7 +14,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.test.ManualClock; -import com.yahoo.test.json.JsonTestHelper; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockLogRetriever; import com.yahoo.vespa.config.server.MockProvisioner; @@ -23,7 +21,8 @@ import com.yahoo.vespa.config.server.MockTesterClient; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; import com.yahoo.vespa.config.server.application.ApplicationReindexing; -import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; +import com.yahoo.vespa.config.server.application.ClusterReindexing; +import com.yahoo.vespa.config.server.application.ClusterReindexing.Status; import com.yahoo.vespa.config.server.application.HttpProxy; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.deploy.DeployTester; @@ -31,6 +30,7 @@ import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.StaticResponse; +import com.yahoo.vespa.config.server.http.v2.ApplicationHandler.ReindexingResponse; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -42,16 +42,17 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import javax.ws.rs.client.Client; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.time.Instant; import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.container.jdisc.HttpRequest.createTestRequest; @@ -437,6 +438,80 @@ public class ApplicationHandlerTest { assertEquals("report", getRenderedString(response)); } + @Test + public void testClusterReindexingStateSerialization() { + Stream.of(ClusterReindexing.State.values()).forEach(ReindexingResponse::toString); + } + + @Test + public void testReindexingSerialization() throws IOException { + Instant now = Instant.ofEpochMilli(123456); + ApplicationReindexing applicationReindexing = ApplicationReindexing.ready(now.minusSeconds(10)) + .withPending("foo", "bar", 123L) + .withReady("moo", now.minusSeconds(1)) + .withReady("moo", "baz", now); + ClusterReindexing clusterReindexing = new ClusterReindexing(Map.of("bax", new Status(now, null, null, null, null), + "baz", new Status(now.plusSeconds(1), + now.plusSeconds(2), + ClusterReindexing.State.FAILED, + "message", + "some"))); + assertJsonEquals(getRenderedString(new ReindexingResponse(applicationReindexing, + Map.of("boo", clusterReindexing, + "moo", clusterReindexing))), + "{\n" + + " \"enabled\": true,\n" + + " \"status\": {\n" + + " \"readyMillis\": 113456\n" + + " },\n" + + " \"clusters\": {\n" + + " \"boo\": {\n" + + " \"pending\": {},\n" + + " \"ready\": {\n" + + " \"bax\": {\n" + + " \"startedMillis\": 123456\n" + + " },\n" + + " \"baz\": {\n" + + " \"startedMillis\": 124456,\n" + + " \"endedMillis\": 125456,\n" + + " \"state\": \"failed\",\n" + + " \"message\": \"message\",\n" + + " \"progress\": \"some\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"foo\": {\n" + + " \"pending\": {\n" + + " \"bar\": 123\n" + + " },\n" + + " \"ready\": {},\n" + + " \"status\": {\n" + + " \"readyMillis\": 113456\n" + + " }\n" + + " },\n" + + " \"moo\": {\n" + + " \"pending\": {},\n" + + " \"ready\": {\n" + + " \"baz\": {\n" + + " \"readyMillis\": 123456,\n" + + " \"startedMillis\": 124456,\n" + + " \"endedMillis\": 125456,\n" + + " \"state\": \"failed\",\n" + + " \"message\": \"message\",\n" + + " \"progress\": \"some\"\n" + + " },\n" + + " \"bax\": {\n" + + " \"startedMillis\": 123456\n" + + " }\n" + + " },\n" + + " \"status\": {\n" + + " \"readyMillis\": 122456\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"); + } + private void assertNotAllowed(Method method) throws IOException { String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default"; deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", @@ -558,21 +633,6 @@ public class ApplicationHandlerTest { return createApplicationHandler().handle(createTestRequest(restartUrl, GET)); } - private static class MockStateApiFactory implements ConfigConvergenceChecker.StateApiFactory { - boolean createdApi = false; - @Override - public ConfigConvergenceChecker.StateApi createStateApi(Client client, URI serviceUri) { - createdApi = true; - return () -> { - try { - return new ObjectMapper().readTree("{\"config\":{\"generation\":1}}"); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - } - } - private ApplicationHandler createApplicationHandler() { return createApplicationHandler(applicationRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 325db1feba6..6729be20305 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; @@ -108,6 +109,17 @@ public class LbServicesProducerTest { } } + @Test + public void use_power_of_two_lb_is_configured_from_feature_flag() throws IOException, SAXException { + RegionName regionName = RegionName.from("us-east-1"); + + LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); + assertFalse(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").usePowerOfTwoChoicesLb()); + + flagSource.withBooleanFlag(Flags.USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING.id(), true); + conf = createModelAndGetLbServicesConfig(regionName); + assertTrue(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").usePowerOfTwoChoicesLb()); + } private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) throws IOException, SAXException { Zone zone = new Zone(Environment.prod, regionName); Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() diff --git a/container-core/src/main/java/com/yahoo/restapi/Path.java b/container-core/src/main/java/com/yahoo/restapi/Path.java index fe65245fd15..23a791e7532 100644 --- a/container-core/src/main/java/com/yahoo/restapi/Path.java +++ b/container-core/src/main/java/com/yahoo/restapi/Path.java @@ -42,24 +42,6 @@ public class Path { private final Map<String, String> values = new HashMap<>(); private String rest = ""; - /** - * @deprecated use {@link #Path(URI)} for correct handling of URL encoded paths. - */ - @Deprecated - public Path(String path) { - this(path, ""); - } - - /** - * @deprecated use {@link #Path(URI, String)} for correct handling of URL encoded paths. - */ - @Deprecated - public Path(String path, String optionalPrefix) { - this.optionalPrefix = optionalPrefix; - this.pathString = path; - this.elements = path.split("/"); - } - public Path(URI uri) { this(uri, ""); } diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index b239518303d..8691d9a7ffb 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -446,7 +446,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.32.v20200930</jetty.version> + <jetty.version>9.4.35.v20201120</jetty.version> <org.lz4.version>1.7.1</org.lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 4614f8f9857..3158c06b0b1 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -261,12 +261,16 @@ public final class ConfiguredApplication implements Application { private void startReconfigurerThread() { reconfigurerThread = new Thread(() -> { + boolean restartOnDeploy = false; while ( ! Thread.interrupted()) { try { ContainerBuilder builder = createBuilderWithGuiceBindings(); + // Restart on deploy is sticky: Once it is set no future generation should be applied until restart + restartOnDeploy = restartOnDeploy || qrConfig.restartOnDeploy(); + // Block until new config arrives, and it should be applied - configurer.getNewComponentGraph(builder.guiceModules().activate(), qrConfig.restartOnDeploy()); + configurer.getNewComponentGraph(builder.guiceModules().activate(), restartOnDeploy); initializeAndActivateContainer(builder); } catch (ConfigInterruptedException e) { break; diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 19fb1862262..cd3f011352d 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -6089,6 +6089,7 @@ "public final java.lang.Object get(java.lang.String, java.util.Map)", "public final java.lang.Object get(java.lang.String, java.util.Map, com.yahoo.processing.request.Properties)", "public final java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", + "public final com.yahoo.search.query.profile.compiled.DimensionalMap getEntries()", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile clone()", "public java.lang.String toString()", "public bridge synthetic com.yahoo.component.AbstractComponent clone()", diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 4995927f7a2..ce31b9a3ba3 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -181,7 +181,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { //---------------- Tracing ---------------------------------------------------- - private static Logger log = Logger.getLogger(Query.class.getName()); + private static final Logger log = Logger.getLogger(Query.class.getName()); /** The time this query was created */ private long startTime; @@ -200,7 +200,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static final CompoundName TIMEOUT = new CompoundName("timeout"); - private static QueryProfileType argumentType; + private static final QueryProfileType argumentType; static { argumentType = new QueryProfileType("native"); argumentType.setBuiltin(true); @@ -226,7 +226,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static QueryProfileType getArgumentType() { return argumentType; } /** The aliases of query properties */ - private static Map<String, CompoundName> propertyAliases; + private static final Map<String, CompoundName> propertyAliases; static { Map<String,CompoundName> propertyAliasesBuilder = new HashMap<>(); addAliases(Query.getArgumentType(), propertyAliasesBuilder); @@ -316,7 +316,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * Creates a query from a request * * @param request the HTTP request from which this is created - * @param queryProfile the query profile to use for this query, or null if none. + * @param queryProfile the query profile to use for this query, or null if none */ public Query(HttpRequest request, CompiledQueryProfile queryProfile) { this(request, request.propertyMap(), queryProfile); @@ -325,9 +325,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Creates a query from a request * - * @param request the HTTP request from which this is created. - * @param requestMap the property map of the query. - * @param queryProfile the query profile to use for this query, or null if none. + * @param request the HTTP request from which this is created + * @param requestMap the property map of the query + * @param queryProfile the query profile to use for this query, or null if none */ public Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile) { super(new QueryPropertyAliases(propertyAliases)); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index c6b0f4a533b..2439908183c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -183,6 +183,11 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable return substitute(value.value(), context, substitution); } + /** Returns all the entries from the profile **/ + public final DimensionalMap<ValueWithSource> getEntries() { + return this.entries; + } + private Object substitute(Object value, Map<String, String> context, Properties substitution) { if (value == null) return value; if (substitution == null) return value; diff --git a/container/pom.xml b/container/pom.xml index cf3aa21513b..4f045aac90e 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -45,6 +45,10 @@ <groupId>io.airlift</groupId> <artifactId>airline</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents.client5</groupId> + <artifactId>httpclient5</artifactId> + </exclusion> </exclusions> </dependency> </dependencies> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java index faa2c39ee65..799bc814abe 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java @@ -14,7 +14,6 @@ import java.util.List; public class RefeedAction { public final String name; - public final boolean allowed; public final String documentType; public final String clusterName; public final List<ServiceInfo> services; @@ -22,13 +21,11 @@ public class RefeedAction { @JsonCreator public RefeedAction(@JsonProperty("name") String name, - @JsonProperty("allowed") boolean allowed, @JsonProperty("documentType") String documentType, @JsonProperty("clusterName") String clusterName, @JsonProperty("services") List<ServiceInfo> services, @JsonProperty("messages") List<String> messages) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; this.services = services; @@ -39,7 +36,6 @@ public class RefeedAction { public String toString() { return "RefeedAction{" + "name='" + name + '\'' + - ", allowed=" + allowed + ", documentType='" + documentType + '\'' + ", clusterName='" + clusterName + '\'' + ", services=" + services + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java index c5735fbd4a6..c2b28a94c66 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java @@ -14,7 +14,6 @@ import java.util.List; public class ReindexAction { public final String name; - public final boolean allowed; public final String documentType; public final String clusterName; public final List<ServiceInfo> services; @@ -22,13 +21,11 @@ public class ReindexAction { @JsonCreator public ReindexAction(@JsonProperty("name") String name, - @JsonProperty("allowed") boolean allowed, @JsonProperty("documentType") String documentType, @JsonProperty("clusterName") String clusterName, @JsonProperty("services") List<ServiceInfo> services, @JsonProperty("messages") List<String> messages) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; this.services = services; @@ -39,7 +36,6 @@ public class ReindexAction { public String toString() { return "ReindexAction{" + "name='" + name + '\'' + - ", allowed=" + allowed + ", documentType='" + documentType + '\'' + ", clusterName='" + clusterName + '\'' + ", services=" + services + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java index 8d002640156..b6c7899aef1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import java.time.Instant; import java.util.Map; import java.util.Objects; +import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -115,31 +116,69 @@ public class ApplicationReindexing { public static class Status { private final Instant readyAt; + private final Instant startedAt; + private final Instant endedAt; + private final State state; + private final String message; + private final String progress; + + public Status(Instant readyAt, Instant startedAt, Instant endedAt, State state, String message, String progress) { + this.readyAt = readyAt; + this.startedAt = startedAt; + this.endedAt = endedAt; + this.state = state; + this.message = message; + this.progress = progress; + } public Status(Instant readyAt) { - this.readyAt = requireNonNull(readyAt); + this(readyAt, null, null, null, null, null); } - public Instant readyAt() { return readyAt; } + public Optional<Instant> readyAt() { return Optional.ofNullable(readyAt); } + public Optional<Instant> startedAt() { return Optional.ofNullable(startedAt); } + public Optional<Instant> endedAt() { return Optional.ofNullable(endedAt); } + public Optional<State> state() { return Optional.ofNullable(state); } + public Optional<String> message() { return Optional.ofNullable(message); } + public Optional<String> progress() { return Optional.ofNullable(progress); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Status status = (Status) o; - return readyAt.equals(status.readyAt); + return Objects.equals(readyAt, status.readyAt) && + Objects.equals(startedAt, status.startedAt) && + Objects.equals(endedAt, status.endedAt) && + state == status.state && + Objects.equals(message, status.message) && + Objects.equals(progress, status.progress); } @Override public int hashCode() { - return Objects.hash(readyAt); + return Objects.hash(readyAt, startedAt, endedAt, state, message, progress); } @Override public String toString() { - return "ready at " + readyAt; + return "Status{" + + "readyAt=" + readyAt + + ", startedAt=" + startedAt + + ", endedAt=" + endedAt + + ", state=" + state + + ", message='" + message + '\'' + + ", progress='" + progress + '\'' + + '}'; } } + + public enum State { + + PENDING, RUNNING, FAILED, SUCCESSFUL; + + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java index fd339e3bb43..98b7ffd1d47 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import java.time.Instant; +import java.util.List; import java.util.Optional; /** @@ -17,19 +19,25 @@ public class Cluster { private final ClusterResources current; private final Optional<ClusterResources> target; private final Optional<ClusterResources> suggested; + private final List<ScalingEvent> scalingEvents; + private final String autoscalingStatus; public Cluster(ClusterSpec.Id id, ClusterResources min, ClusterResources max, ClusterResources current, Optional<ClusterResources> target, - Optional<ClusterResources> suggested) { + Optional<ClusterResources> suggested, + List<ScalingEvent> scalingEvents, + String autoscalingStatus) { this.id = id; this.min = min; this.max = max; this.current = current; this.target = target; this.suggested = suggested; + this.scalingEvents = scalingEvents; + this.autoscalingStatus = autoscalingStatus; } public ClusterSpec.Id id() { return id; } @@ -38,10 +46,29 @@ public class Cluster { public ClusterResources current() { return current; } public Optional<ClusterResources> target() { return target; } public Optional<ClusterResources> suggested() { return suggested; } + public List<ScalingEvent> scalingEvents() { return scalingEvents; } + public String autoscalingStatus() { return autoscalingStatus; } @Override public String toString() { return "cluster '" + id + "'"; } + public static class ScalingEvent { + + private final ClusterResources from, to; + private final Instant at; + + public ScalingEvent(ClusterResources from, ClusterResources to, Instant at) { + this.from = from; + this.to = to; + this.at = at; + } + + public ClusterResources from() { return from; } + public ClusterResources to() { return to; } + public Instant at() { return at; } + + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index 8fd294f64f8..7d85c11789b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -14,10 +14,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeHist import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * A node in hosted Vespa. @@ -57,6 +59,8 @@ public class Node { private final Optional<ApplicationId> exclusiveTo; private final Map<String, JsonNode> reports; private final List<NodeHistory> history; + private final Set<String> additionalIpAddresses; + private final String openStackId; public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, @@ -64,7 +68,8 @@ public class Node { Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, - DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports, List<NodeHistory> history) { + DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports, List<NodeHistory> history, + Set<String> additionalIpAddresses, String openStackId) { this.hostname = hostname; this.parentHostname = parentHostname; this.state = state; @@ -95,6 +100,8 @@ public class Node { this.currentDockerImage = currentDockerImage; this.reports = reports; this.history = history; + this.openStackId = openStackId; + this.additionalIpAddresses = additionalIpAddresses; } public HostName hostname() { @@ -211,6 +218,14 @@ public class Node { return history; } + public Set<String> additionalIpAddresses() { + return additionalIpAddresses; + } + + public String openStackId() { + return openStackId; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -285,6 +300,8 @@ public class Node { private Optional<ApplicationId> exclusiveTo = Optional.empty(); private Map<String, JsonNode> reports = new HashMap<>(); private List<NodeHistory> history = new ArrayList<>(); + private Set<String> additionalIpAddresses = new HashSet<>(); + private String openStackId; public Builder() { } @@ -319,6 +336,8 @@ public class Node { this.exclusiveTo = node.exclusiveTo; this.reports = node.reports; this.history = node.history; + this.additionalIpAddresses = node.additionalIpAddresses; + this.openStackId = node.openStackId; } public Builder hostname(HostName hostname) { @@ -466,12 +485,22 @@ public class Node { return this; } + public Builder additionalIpAddresses(Set<String> additionalIpAddresses) { + this.additionalIpAddresses = additionalIpAddresses; + return this; + } + + public Builder openStackId(String openStackId) { + this.openStackId = openStackId; + return this; + } + public Node build() { return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo, exclusiveTo, - wantedDockerImage, currentDockerImage, reports, history); + wantedDockerImage, currentDockerImage, reports, history, additionalIpAddresses, openStackId); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index ca8af48e4fd..af1b3fa53fc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -135,7 +135,9 @@ public interface NodeRepository { dockerImageFrom(node.getWantedDockerImage()), dockerImageFrom(node.getCurrentDockerImage()), node.getReports(), - node.getHistory()); + node.getHistory(), + node.getAdditionalIpAddresses(), + node.getOpenStackId()); } private static String clusterIdOf(NodeMembership nodeMembership) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java index 6054e05149b..5c946538625 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java @@ -15,7 +15,6 @@ import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) public class PrepareResponse { public TenantId tenant; - @JsonProperty("activate") public URI activationUri; public String message; public List<Log> log; public ConfigChangeActions configChangeActions; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 700be6d263a..efdeff8fc16 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -144,6 +144,9 @@ public enum JobType { Map.of(Public, ZoneId.from("dev", "aws-us-east-1c"), PublicCd, ZoneId.from("dev", "aws-us-east-1c"))), + perfAwsUsEast1c ("perf-aws-us-east-1c", + Map.of(Public, ZoneId.from("perf", "aws-us-east-1c"))), + perfUsEast3 ("perf-us-east-3", Map.of(main, ZoneId.from("perf" , "us-east-3"))); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java index 298928a881d..99a72fbc827 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java @@ -7,7 +7,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * @author bratseth @@ -26,6 +28,10 @@ public class ClusterData { public ClusterResourcesData suggested; @JsonProperty("target") public ClusterResourcesData target; + @JsonProperty("scalingEvents") + public List<ScalingEventData> scalingEvents; + @JsonProperty("autoscalingStatus") + public String autoscalingStatus; public Cluster toCluster(String id) { return new Cluster(ClusterSpec.Id.from(id), @@ -33,7 +39,10 @@ public class ClusterData { max.toClusterResources(), current.toClusterResources(), target == null ? Optional.empty() : Optional.of(target.toClusterResources()), - suggested == null ? Optional.empty() : Optional.of(suggested.toClusterResources())); + suggested == null ? Optional.empty() : Optional.of(suggested.toClusterResources()), + scalingEvents == null ? List.of() + : scalingEvents.stream().map(data -> data.toScalingEvent()).collect(Collectors.toList()), + autoscalingStatus); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java index 97e7f2e897a..393814478dd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class NodeHistory { + @JsonProperty("at") public Long at; @JsonProperty("agent") diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 7bb47185751..65d6f2a5fa6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -29,6 +28,8 @@ public class NodeRepositoryNode { private Set<String> ipAddresses; @JsonProperty("additionalIpAddresses") private Set<String> additionalIpAddresses; + @JsonProperty("additionalHostnames") + private List<String> additionalHostnames; @JsonProperty("openStackId") private String openStackId; @JsonProperty("flavor") @@ -142,6 +143,14 @@ public class NodeRepositoryNode { this.additionalIpAddresses = additionalIpAddresses; } + public List<String> getAdditionalHostnames() { + return additionalHostnames; + } + + public void setAdditionalHostnames(List<String> additionalHostnames) { + this.additionalHostnames = additionalHostnames; + } + public String getOpenStackId() { return openStackId; } @@ -397,6 +406,7 @@ public class NodeRepositoryNode { ", hostname='" + hostname + '\'' + ", ipAddresses=" + ipAddresses + ", additionalIpAddresses=" + additionalIpAddresses + + ", additionalHostnames=" + additionalHostnames + ", openStackId='" + openStackId + '\'' + ", flavor='" + flavor + '\'' + ", resources=" + resources + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ScalingEventData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ScalingEventData.java new file mode 100644 index 00000000000..b33a7436522 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ScalingEventData.java @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; + +import java.time.Instant; + +/** + * @author bratseth + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ScalingEventData { + + @JsonProperty("from") + public ClusterResourcesData from; + + @JsonProperty("to") + public ClusterResourcesData to; + + @JsonProperty("at") + public Long at; + + public Cluster.ScalingEvent toScalingEvent() { + return new Cluster.ScalingEvent(from.toClusterResources(), to.toClusterResources(), Instant.ofEpochMilli(at)); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index c3126cc8b7a..85db447dfbd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -92,7 +92,8 @@ public enum RoleDefinition { paymentProcessor(Policy.paymentProcessor), hostedAccountant(Policy.hostedAccountant, - Policy.collectionMethodUpdate); + Policy.collectionMethodUpdate, + Policy.planUpdate); private final Set<RoleDefinition> parents; private final Set<Policy> policies; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 10d4732984c..1a24b5361dd 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -172,7 +172,7 @@ public class RoleTest { } @Test - public void billing() { + public void billing_tenant() { URI billing = URI.create("/billing/v1/tenant/t1/billing"); Role user = Role.reader(TenantName.from("t1")); @@ -188,4 +188,129 @@ public class RoleTest { } + @Test + public void billing_test() { + var tester = new EnforcerTester(publicCdEnforcer); + + var accountant = Role.hostedAccountant(); + var operator = Role.hostedOperator(); + var reader = Role.reader(TenantName.from("t1")); + var developer = Role.developer(TenantName.from("t1")); + var admin = Role.administrator(TenantName.from("t1")); + var otherAdmin = Role.administrator(TenantName.from("t2")); + + tester.on("/billing/v1/tenant/t1/token") + .assertAction(accountant) + .assertAction(operator, Action.create, Action.read, Action.update, Action.delete) + .assertAction(reader) + .assertAction(developer) + .assertAction(admin, Action.read) + .assertAction(otherAdmin); + + tester.on("/billing/v1/tenant/t1/instrument") + .assertAction(accountant) + .assertAction(operator, Action.create, Action.read, Action.update, Action.delete) + .assertAction(reader, Action.read, Action.delete) + .assertAction(developer, Action.read, Action.delete) + .assertAction(admin, Action.read, Action.update, Action.delete) + .assertAction(otherAdmin); + + tester.on("/billing/v1/tenant/t1/instrument/i1") + .assertAction(accountant) + .assertAction(operator, Action.create, Action.read, Action.update, Action.delete) + .assertAction(reader, Action.read, Action.delete) + .assertAction(developer, Action.read, Action.delete) + .assertAction(admin, Action.read, Action.update, Action.delete) + .assertAction(otherAdmin); + + tester.on("/billing/v1/tenant/t1/billing") + .assertAction(accountant) + .assertAction(operator, Action.create, Action.read, Action.update, Action.delete) + .assertAction(reader, Action.read) + .assertAction(developer, Action.read) + .assertAction(admin, Action.read) + .assertAction(otherAdmin); + + tester.on("/billing/v1/tenant/t1/plan") + .assertAction(accountant, Action.update) + .assertAction(operator, Action.create, Action.read, Action.update, Action.delete) + .assertAction(reader) + .assertAction(developer) + .assertAction(admin, Action.update) + .assertAction(otherAdmin); + + tester.on("/billing/v1/tenant/t1/collection") + .assertAction(accountant, Action.update) + .assertAction(operator, Action.create, Action.read, Action.update, Action.delete) + .assertAction(reader) + .assertAction(developer) + .assertAction(admin) + .assertAction(otherAdmin); + + tester.on("/billing/v1/billing") + .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) + .assertAction(operator, Action.read) + .assertAction(reader) + .assertAction(developer) + .assertAction(admin) + .assertAction(otherAdmin); + + tester.on("/billing/v1/invoice/tenant/t1/line-item") + .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) + .assertAction(operator, Action.read) + .assertAction(reader) + .assertAction(developer) + .assertAction(admin) + .assertAction(otherAdmin); + + tester.on("/billing/v1/invoice") + .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) + .assertAction(operator, Action.read) + .assertAction(reader) + .assertAction(developer) + .assertAction(admin) + .assertAction(otherAdmin); + + tester.on("/billing/v1/invoice/i1/status") + .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) + .assertAction(operator, Action.read) + .assertAction(reader) + .assertAction(developer) + .assertAction(admin) + .assertAction(otherAdmin); + } + + private static class EnforcerTester { + private final Enforcer enforcer; + private final URI resource; + + EnforcerTester(Enforcer enforcer) { + this(enforcer, null); + } + + EnforcerTester(Enforcer enforcer, URI uri) { + this.enforcer = enforcer; + this.resource = uri; + } + + public EnforcerTester on(String uri) { + return new EnforcerTester(enforcer, URI.create(uri)); + } + + public EnforcerTester assertAction(Role role, Action ...actions) { + var allowed = List.of(actions); + + allowed.forEach(action -> { + var msg = String.format("%s should be allowed to %s on %s", role, action, resource); + assertTrue(msg, enforcer.allows(role, action, resource)); + }); + + Action.all().stream().filter(a -> ! allowed.contains(a)).forEach(action -> { + var msg = String.format("%s should not be allowed to %s on %s", role, action, resource); + assertFalse(msg, enforcer.allows(role, action, resource)); + }); + + return this; + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 1e28ed466e8..ac146858145 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.log.LogLevel; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.flags.BooleanFlag; @@ -156,7 +155,7 @@ public class EndpointCertificateManager { curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { - log.log(LogLevel.INFO, "Cert for app " + applicationId.serializedForm() + log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + " has not been requested in a month and app has no deployments" + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : "")); if (mode == CleanupMode.ENABLE) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 9b959bf1765..ed1e442f266 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -219,43 +219,11 @@ public class InternalStepRunner implements StepRunner { LogEntry.typeOf(LogLevel.parse(entry.level)), entry.message)) .collect(toList())); - if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { - List<String> messages = new ArrayList<>(); - messages.add("Deploy failed due to non-compatible changes that require re-feed."); - messages.add("Your options are:"); - messages.add("1. Revert the incompatible changes."); - messages.add("2. If you think it is safe in your case, you can override this validation, see"); - messages.add(" http://docs.vespa.ai/documentation/reference/validation-overrides.html"); - messages.add("3. Deploy as a new application under a different name."); - messages.add("Illegal actions:"); - prepareResponse.configChangeActions.refeedActions.stream() - .filter(action -> ! action.allowed) - .flatMap(action -> action.messages.stream()) - .forEach(messages::add); - logger.log(messages); - return Optional.of(deploymentFailed); - } - - if ( ! prepareResponse.configChangeActions.reindexActions.stream().allMatch(action -> action.allowed)) { - List<String> messages = new ArrayList<>(); - messages.add("Deploy failed due to non-compatible changes that require re-index."); - messages.add("Your options are:"); - messages.add("1. Revert the incompatible changes."); - messages.add("2. If you think it is safe in your case, you can override this validation, see"); - messages.add(" http://docs.vespa.ai/documentation/reference/validation-overrides.html"); - messages.add("3. Deploy as a new application under a different name."); - messages.add("Illegal actions:"); - prepareResponse.configChangeActions.reindexActions.stream() - .filter(action -> ! action.allowed) - .flatMap(action -> action.messages.stream()) - .forEach(messages::add); - logger.log(messages); - return Optional.of(deploymentFailed); - } logger.log("Deployment successful."); if (prepareResponse.message != null) logger.log(prepareResponse.message); + return Optional.of(running); } catch (ConfigServerException e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 4d928a6b8a7..29e9ec53577 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -701,6 +701,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { toSlime(cluster.current(), clusterObject.setObject("current")); cluster.target().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); cluster.suggested().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); + scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); + clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); } return new SlimeJsonResponse(slime); } @@ -1593,7 +1595,22 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } void setStatus(Cursor statusObject, ApplicationReindexing.Status status) { - statusObject.setLong("readyAtMillis", status.readyAt().toEpochMilli()); + status.readyAt().ifPresent(readyAt -> statusObject.setLong("readyAtMillis", readyAt.toEpochMilli())); + status.startedAt().ifPresent(startedAt -> statusObject.setLong("startedAtMillis", startedAt.toEpochMilli())); + status.endedAt().ifPresent(endedAt -> statusObject.setLong("endedAtMillis", endedAt.toEpochMilli())); + status.state().map(ApplicationApiHandler::toString).ifPresent(state -> statusObject.setString("state", state)); + status.message().ifPresent(message -> statusObject.setString("message", message)); + status.progress().ifPresent(progress -> statusObject.setString("progress", progress)); + } + + private static String toString(ApplicationReindexing.State state) { + switch (state) { + case PENDING: return "pending"; + case RUNNING: return "running"; + case FAILED: return "failed"; + case SUCCESSFUL: return "successful"; + default: return null; + } } /** Enables reindexing of an application in a zone. */ @@ -1914,6 +1931,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { object.setDouble("cost", Math.round(resources.nodes() * resources.nodeResources().cost() * 100.0 / 3.0) / 100.0); } + private void scalingEventsToSlime(List<Cluster.ScalingEvent> scalingEvents, Cursor scalingEventsArray) { + for (Cluster.ScalingEvent scalingEvent : scalingEvents) { + Cursor scalingEventObject = scalingEventsArray.addObject(); + toSlime(scalingEvent.from(), scalingEventObject.setObject("from")); + toSlime(scalingEvent.to(), scalingEventObject.setObject("to")); + scalingEventObject.setLong("at", scalingEvent.at().toEpochMilli()); + } + } + private void toSlime(NodeResources resources, Cursor object) { object.setDouble("vcpu", resources.vcpu()); object.setDouble("memoryGb", resources.memoryGb()); @@ -2054,7 +2080,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { for (RefeedAction refeedAction : result.prepareResponse().configChangeActions.refeedActions) { Cursor refeedActionObject = refeedActionsArray.addObject(); refeedActionObject.setString("name", refeedAction.name); - refeedActionObject.setBool("allowed", refeedAction.allowed); refeedActionObject.setString("documentType", refeedAction.documentType); refeedActionObject.setString("clusterName", refeedAction.clusterName); serviceInfosToSlime(refeedAction.services, refeedActionObject.setArray("services")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 7b248052eac..d54971f5b1d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -125,34 +125,7 @@ public class InternalStepRunnerTest { } @Test - public void reindexRequirementBlocksDeployment() { - tester.configServer().setConfigChangeActions(new ConfigChangeActions(List.of(), - List.of(), - List.of(new ReindexAction("Reindex", - false, - "doctype", - "cluster", - Collections.emptyList(), - List.of("Reindex it!"))))); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); - } - - @Test - public void refeedRequirementBlocksDeployment() { - tester.configServer().setConfigChangeActions(new ConfigChangeActions(List.of(), - List.of(new RefeedAction("Refeed", - false, - "doctype", - "cluster", - Collections.emptyList(), - singletonList("Refeed it!"))), - List.of())); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); - } - - @Test + // TODO jonmv: Change to only wait for restarts, and remove triggering of restarts from runner. public void restartsServicesAndWaitsForRestartAndReboot() { RunId id = app.newRun(JobType.productionUsCentral1); ZoneId zone = id.type().zone(system()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 8acce352d5a..2b9bad4899f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.documentapi.ProgressToken; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; @@ -108,12 +109,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Assigns a reserved tenant node to the given deployment, with initial versions. */ public void provision(ZoneId zone, ApplicationId application, ClusterSpec.Id clusterId) { + var current = new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1, slow, remote)); Cluster cluster = new Cluster(clusterId, new ClusterResources(2, 1, new NodeResources(1, 4, 20, 1, slow, remote)), new ClusterResources(2, 1, new NodeResources(4, 16, 90, 1, slow, remote)), - new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1, slow, remote)), + current, Optional.of(new ClusterResources(2, 1, new NodeResources(3, 8, 50, 1, slow, remote))), - Optional.empty()); + Optional.empty(), + List.of(new Cluster.ScalingEvent(new ClusterResources(0, 0, NodeResources.unspecified()), + current, + Instant.ofEpochMilli(1234))), + "the autoscaling status"); nodeRepository.putApplication(zone, new com.yahoo.vespa.hosted.controller.api.integration.configserver.Application(application, List.of(cluster))); @@ -425,9 +431,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer Map.of("cluster", new ApplicationReindexing.Cluster(new Status(Instant.ofEpochMilli(234)), Map.of("type", 100L), - Map.of("type", new Status(Instant.ofEpochMilli(345))))))); + Map.of("type", new Status(Instant.ofEpochMilli(345), + Instant.ofEpochMilli(456), + Instant.ofEpochMilli(567), + ApplicationReindexing.State.FAILED, + "(＃｀д´)ﾉ", + "some")))))); + + } + @Override public void disableReindexing(DeploymentId deployment) { } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 5e98ac0d3ee..d303d1a9b83 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -608,7 +608,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET to get reindexing status tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindexing", GET) .userIdentity(USER_ID), - "{\"enabled\":true,\"status\":{\"readyAtMillis\":123},\"clusters\":[{\"name\":\"cluster\",\"status\":{\"readyAtMillis\":234},\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345}]}]}"); + "{\"enabled\":true,\"status\":{\"readyAtMillis\":123},\"clusters\":[{\"name\":\"cluster\",\"status\":{\"readyAtMillis\":234},\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345,\"startedAtMillis\":456,\"endedAtMillis\":567,\"state\":\"failed\",\"message\":\"(＃｀д´)ﾉ\",\"progress\":\"some\"}]}]}"); // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index 65fa2a4bf70..817cee7732a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -52,7 +52,39 @@ "storageType": "remote" }, "cost": "(ignore)" - } + }, + "scalingEvents": [ + { + "from": { + "nodes": 0, + "groups": 0, + "nodeResources": { + "vcpu": 0.0, + "memoryGb": 0.0, + "diskGb": 0.0, + "bandwidthGbps": 0.0, + "diskSpeed": "fast", + "storageType": "any" + }, + "cost": "(ignore)" + }, + "to": { + "nodes": 2, + "groups": 1, + "nodeResources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "slow", + "storageType": "remote" + }, + "cost": "(ignore)" + }, + "at": 1234 + } + ], + "autoscalingStatus": "the autoscaling status" } } }
\ No newline at end of file diff --git a/document/src/test/resources/tensor/multi_cell_tensor__cpp b/document/src/test/resources/tensor/multi_cell_tensor__cpp Binary files differindex 9adda236a4a..deb53463fb5 100644 --- a/document/src/test/resources/tensor/multi_cell_tensor__cpp +++ b/document/src/test/resources/tensor/multi_cell_tensor__cpp diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.h b/document/src/vespa/document/serialization/vespadocumentdeserializer.h index 6792914d9da..5819c6a23cf 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.h @@ -7,7 +7,7 @@ #include <memory> namespace vespalib { class nbostream; } -namespace vespalib::eval { class Value; } +namespace vespalib::eval { struct Value; } namespace document { class DocumentId; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index f83f31506e4..d420d64f461 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -145,14 +145,13 @@ public class Messages60TestCase extends MessagesTestBase { @Override public void run() { GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); - msg.setLoadType(loadTypes.getNameMap().get("foo")); msg.setBucketSpace(BUCKET_SPACE); assertEquals(BASE_MESSAGE_LENGTH + 12 + serializedLength(BUCKET_SPACE), serialize("GetBucketListMessage", msg)); for (Language lang : LANGUAGES) { msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("foo", msg.getLoadType().getName()); + assertEquals("default", msg.getLoadType().getName()); assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } } diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index c7bb1015e02..5d25002021b 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -89,16 +89,13 @@ bool Messages60Test::testGetBucketListMessage() { GetBucketListMessage msg(document::BucketId(16, 123)); - msg.setLoadType(_loadTypes["foo"]); msg.setBucketSpace("beartato"); - EXPECT_EQUAL(string("foo"), msg.getLoadType().getName()); EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u + serializedLength("beartato"), serialize("GetBucketListMessage", msg)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { mbus::Routable::UP obj = deserialize("GetBucketListMessage", DocumentProtocol::MESSAGE_GETBUCKETLIST, lang); if (EXPECT_TRUE(obj)) { GetBucketListMessage &ref = static_cast<GetBucketListMessage&>(*obj); - EXPECT_EQUAL(string("foo"), ref.getLoadType().getName()); EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); EXPECT_EQUAL("beartato", ref.getBucketSpace()); } @@ -323,6 +320,7 @@ Messages60Test::testGetDocumentMessage() { GetDocumentMessage tmp(document::DocumentId("id:ns:testdoc::"), "foo bar"); + EXPECT_EQUAL(280u, sizeof(GetDocumentMessage)); EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)31, serialize("GetDocumentMessage", tmp)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { @@ -400,6 +398,11 @@ Messages60Test::testPutDocumentMessage() msg.setTimestamp(666); msg.setCondition(TestAndSetCondition("There's just one condition")); + EXPECT_EQUAL(64u, sizeof(vespalib::string)); + EXPECT_EQUAL(sizeof(std::string), sizeof(TestAndSetCondition)); + EXPECT_EQUAL(112u, sizeof(DocumentMessage)); + EXPECT_EQUAL(sizeof(TestAndSetCondition) + sizeof(DocumentMessage), sizeof(TestAndSetMessage)); + EXPECT_EQUAL(sizeof(TestAndSetMessage) + 24, sizeof(PutDocumentMessage)); EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 45u + serializedLength(msg.getCondition().getSelection()), @@ -447,6 +450,7 @@ Messages60Test::testPutDocumentReply() reply.setHighestModificationTimestamp(30); EXPECT_EQUAL(13u, serialize("PutDocumentReply", reply)); + EXPECT_EQUAL(112u, sizeof(WriteDocumentReply)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { mbus::Routable::UP obj = deserialize("PutDocumentReply", DocumentProtocol::REPLY_PUTDOCUMENT, lang); @@ -466,6 +470,7 @@ Messages60Test::testUpdateDocumentReply() reply.setHighestModificationTimestamp(30); EXPECT_EQUAL(14u, serialize("UpdateDocumentReply", reply)); + EXPECT_EQUAL(120u, sizeof(UpdateDocumentReply)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { mbus::Routable::UP obj = deserialize("UpdateDocumentReply", DocumentProtocol::REPLY_UPDATEDOCUMENT, lang); @@ -485,12 +490,13 @@ Messages60Test::testRemoveDocumentMessage() msg.setCondition(TestAndSetCondition("There's just one condition")); + EXPECT_EQUAL(sizeof(TestAndSetMessage) + 104, sizeof(RemoveDocumentMessage)); EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(20) + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { auto routablePtr = deserialize("RemoveDocumentMessage", DocumentProtocol::MESSAGE_REMOVEDOCUMENT, lang); - if (EXPECT_TRUE(routablePtr.get() != nullptr)) { + if (EXPECT_TRUE(routablePtr)) { auto & ref = static_cast<RemoveDocumentMessage &>(*routablePtr); EXPECT_EQUAL(string("id:ns:testdoc::"), ref.getDocumentId().toString()); EXPECT_EQUAL(msg.getCondition().getSelection(), ref.getCondition().getSelection()); @@ -506,6 +512,7 @@ Messages60Test::testRemoveDocumentReply() std::vector<uint64_t> ts; reply.setWasFound(false); reply.setHighestModificationTimestamp(30); + EXPECT_EQUAL(120u, sizeof(RemoveDocumentReply)); EXPECT_EQUAL(14u, serialize("RemoveDocumentReply", reply)); @@ -663,12 +670,13 @@ Messages60Test::testUpdateDocumentMessage() msg.setNewTimestamp(777u); msg.setCondition(TestAndSetCondition("There's just one condition")); + EXPECT_EQUAL(sizeof(TestAndSetMessage) + 32, sizeof(UpdateDocumentMessage)); EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 93u + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { auto routableUp = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); - if (EXPECT_TRUE(routableUp.get() != nullptr)) { + if (EXPECT_TRUE(routableUp)) { auto & deserializedMsg = static_cast<UpdateDocumentMessage &>(*routableUp); EXPECT_EQUAL(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate()); EXPECT_EQUAL(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); @@ -885,6 +893,7 @@ Messages60Test::testGetDocumentReply() createDoc(getTypeRepo(), "testdoc", "id:ns:testdoc::"); GetDocumentReply tmp(doc); + EXPECT_EQUAL(128u, sizeof(GetDocumentReply)); EXPECT_EQUAL((size_t)47, serialize("GetDocumentReply", tmp)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { diff --git a/documentapi/src/tests/replymerger/replymerger_test.cpp b/documentapi/src/tests/replymerger/replymerger_test.cpp index 4626ccd0a60..f74ad23da1d 100644 --- a/documentapi/src/tests/replymerger/replymerger_test.cpp +++ b/documentapi/src/tests/replymerger/replymerger_test.cpp @@ -8,6 +8,7 @@ #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> #include <vespa/documentapi/messagebus/messages/getdocumentreply.h> #include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/error.h> using namespace documentapi; diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp index cde15776b62..42aa79332db 100644 --- a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp +++ b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp @@ -38,7 +38,7 @@ LoadTypeSet::LoadTypeSet(const LoadTypeConfig& config) configure(config); } -LoadTypeSet::~LoadTypeSet() { } +LoadTypeSet::~LoadTypeSet() = default; void LoadTypeSet::addLoadType(uint32_t id, const string& name, Priority::Value priority) { diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index dcfc0fa5f6e..a957ce5e4ff 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -8,6 +8,7 @@ #include <vespa/document/util/stringutil.h> #include <vespa/documentapi/documentapi.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/messagebus/error.h> #include <sstream> #include <cassert> @@ -31,16 +32,16 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, string cfg = (configId.empty() ? "client" : configId); // When adding factories to this list, please KEEP THEM ORDERED alphabetically like they are now. - putRoutingPolicyFactory("AND", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::AndPolicyFactory())); - putRoutingPolicyFactory("Content", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::ContentPolicyFactory())); - putRoutingPolicyFactory("MessageType", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::MessageTypePolicyFactory())); - putRoutingPolicyFactory("DocumentRouteSelector", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::DocumentRouteSelectorPolicyFactory(*_repo, cfg))); - putRoutingPolicyFactory("Extern", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::ExternPolicyFactory())); - putRoutingPolicyFactory("LocalService", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::LocalServicePolicyFactory())); - putRoutingPolicyFactory("RoundRobin", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::RoundRobinPolicyFactory())); - putRoutingPolicyFactory("Storage", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::StoragePolicyFactory())); - putRoutingPolicyFactory("SubsetService", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::SubsetServicePolicyFactory())); - putRoutingPolicyFactory("LoadBalancer", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::LoadBalancerPolicyFactory())); + putRoutingPolicyFactory("AND", std::make_shared<RoutingPolicyFactories::AndPolicyFactory>()); + putRoutingPolicyFactory("Content", std::make_shared<RoutingPolicyFactories::ContentPolicyFactory>()); + putRoutingPolicyFactory("MessageType", std::make_shared<RoutingPolicyFactories::MessageTypePolicyFactory>()); + putRoutingPolicyFactory("DocumentRouteSelector", std::make_shared<RoutingPolicyFactories::DocumentRouteSelectorPolicyFactory>(*_repo, cfg)); + putRoutingPolicyFactory("Extern", std::make_shared<RoutingPolicyFactories::ExternPolicyFactory>()); + putRoutingPolicyFactory("LocalService", std::make_shared<RoutingPolicyFactories::LocalServicePolicyFactory>()); + putRoutingPolicyFactory("RoundRobin", std::make_shared<RoutingPolicyFactories::RoundRobinPolicyFactory>()); + putRoutingPolicyFactory("Storage", std::make_shared<RoutingPolicyFactories::StoragePolicyFactory>()); + putRoutingPolicyFactory("SubsetService", std::make_shared<RoutingPolicyFactories::SubsetServicePolicyFactory>()); + putRoutingPolicyFactory("LoadBalancer", std::make_shared<RoutingPolicyFactories::LoadBalancerPolicyFactory>()); // Prepare version specifications to use when adding routable factories. vespalib::VersionSpecification version6(6, 221); @@ -48,42 +49,42 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, std::vector<vespalib::VersionSpecification> from6 = { version6 }; // Add 6.x serialization - putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorMessageFactory()), from6); - putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories60::DestroyVisitorMessageFactory()), from6); - putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories60::DocumentListMessageFactory(*_repo)), from6); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories60::DocumentSummaryMessageFactory()), from6); - putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories60::EmptyBucketsMessageFactory()), from6); - putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories60::GetBucketListMessageFactory()), from6); - putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories60::GetBucketStateMessageFactory()), from6); - putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::GetDocumentMessageFactory()), from6); - putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories60::MapVisitorMessageFactory()), from6); - putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::PutDocumentMessageFactory(*_repo)), from6); - putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories60::QueryResultMessageFactory()), from6); - putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::RemoveDocumentMessageFactory()), from6); - putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories60::RemoveLocationMessageFactory(*_repo)), from6); - putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories60::SearchResultMessageFactory()), from6); - putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories60::StatBucketMessageFactory()), from6); - putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::UpdateDocumentMessageFactory(*_repo)), from6); - putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories60::VisitorInfoMessageFactory()), from6); - putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorReplyFactory()), from6); - putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories60::DestroyVisitorReplyFactory()), from6); - putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories60::DocumentIgnoredReplyFactory()), from6); - putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories60::DocumentListReplyFactory()), from6); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories60::DocumentSummaryReplyFactory()), from6); - putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories60::EmptyBucketsReplyFactory()), from6); - putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories60::GetBucketListReplyFactory()), from6); - putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories60::GetBucketStateReplyFactory()), from6); - putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::GetDocumentReplyFactory(*_repo)), from6); - putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories60::MapVisitorReplyFactory()), from6); - putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::PutDocumentReplyFactory()), from6); - putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories60::QueryResultReplyFactory()), from6); - putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::RemoveDocumentReplyFactory()), from6); - putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories60::RemoveLocationReplyFactory()), from6); - putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories60::SearchResultReplyFactory()), from6); - putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories60::StatBucketReplyFactory()), from6); - putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::UpdateDocumentReplyFactory()), from6); - putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories60::VisitorInfoReplyFactory()), from6); - putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories60::WrongDistributionReplyFactory()), from6); + putRoutableFactory(MESSAGE_CREATEVISITOR, std::make_shared<RoutableFactories60::CreateVisitorMessageFactory>(), from6); + putRoutableFactory(MESSAGE_DESTROYVISITOR, std::make_shared<RoutableFactories60::DestroyVisitorMessageFactory>(), from6); + putRoutableFactory(MESSAGE_DOCUMENTLIST, std::make_shared<RoutableFactories60::DocumentListMessageFactory>(*_repo), from6); + putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, std::make_shared<RoutableFactories60::DocumentSummaryMessageFactory>(), from6); + putRoutableFactory(MESSAGE_EMPTYBUCKETS, std::make_shared<RoutableFactories60::EmptyBucketsMessageFactory>(), from6); + putRoutableFactory(MESSAGE_GETBUCKETLIST, std::make_shared<RoutableFactories60::GetBucketListMessageFactory>(), from6); + putRoutableFactory(MESSAGE_GETBUCKETSTATE, std::make_shared<RoutableFactories60::GetBucketStateMessageFactory>(), from6); + putRoutableFactory(MESSAGE_GETDOCUMENT, std::make_shared<RoutableFactories60::GetDocumentMessageFactory>(), from6); + putRoutableFactory(MESSAGE_MAPVISITOR, std::make_shared<RoutableFactories60::MapVisitorMessageFactory>(), from6); + putRoutableFactory(MESSAGE_PUTDOCUMENT, std::make_shared<RoutableFactories60::PutDocumentMessageFactory>(*_repo), from6); + putRoutableFactory(MESSAGE_QUERYRESULT, std::make_shared<RoutableFactories60::QueryResultMessageFactory>(), from6); + putRoutableFactory(MESSAGE_REMOVEDOCUMENT, std::make_shared<RoutableFactories60::RemoveDocumentMessageFactory>(), from6); + putRoutableFactory(MESSAGE_REMOVELOCATION, std::make_shared<RoutableFactories60::RemoveLocationMessageFactory>(*_repo), from6); + putRoutableFactory(MESSAGE_SEARCHRESULT, std::make_shared<RoutableFactories60::SearchResultMessageFactory>(), from6); + putRoutableFactory(MESSAGE_STATBUCKET, std::make_shared<RoutableFactories60::StatBucketMessageFactory>(), from6); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, std::make_shared<RoutableFactories60::UpdateDocumentMessageFactory>(*_repo), from6); + putRoutableFactory(MESSAGE_VISITORINFO, std::make_shared<RoutableFactories60::VisitorInfoMessageFactory>(), from6); + putRoutableFactory(REPLY_CREATEVISITOR, std::make_shared<RoutableFactories60::CreateVisitorReplyFactory>(), from6); + putRoutableFactory(REPLY_DESTROYVISITOR, std::make_shared<RoutableFactories60::DestroyVisitorReplyFactory>(), from6); + putRoutableFactory(REPLY_DOCUMENTIGNORED, std::make_shared<RoutableFactories60::DocumentIgnoredReplyFactory>(), from6); + putRoutableFactory(REPLY_DOCUMENTLIST, std::make_shared<RoutableFactories60::DocumentListReplyFactory>(), from6); + putRoutableFactory(REPLY_DOCUMENTSUMMARY, std::make_shared<RoutableFactories60::DocumentSummaryReplyFactory>(), from6); + putRoutableFactory(REPLY_EMPTYBUCKETS, std::make_shared<RoutableFactories60::EmptyBucketsReplyFactory>(), from6); + putRoutableFactory(REPLY_GETBUCKETLIST, std::make_shared<RoutableFactories60::GetBucketListReplyFactory>(), from6); + putRoutableFactory(REPLY_GETBUCKETSTATE, std::make_shared<RoutableFactories60::GetBucketStateReplyFactory>(), from6); + putRoutableFactory(REPLY_GETDOCUMENT, std::make_shared<RoutableFactories60::GetDocumentReplyFactory>(*_repo), from6); + putRoutableFactory(REPLY_MAPVISITOR, std::make_shared<RoutableFactories60::MapVisitorReplyFactory>(), from6); + putRoutableFactory(REPLY_PUTDOCUMENT, std::make_shared<RoutableFactories60::PutDocumentReplyFactory>(), from6); + putRoutableFactory(REPLY_QUERYRESULT, std::make_shared<RoutableFactories60::QueryResultReplyFactory>(), from6); + putRoutableFactory(REPLY_REMOVEDOCUMENT, std::make_shared<RoutableFactories60::RemoveDocumentReplyFactory>(), from6); + putRoutableFactory(REPLY_REMOVELOCATION, std::make_shared<RoutableFactories60::RemoveLocationReplyFactory>(), from6); + putRoutableFactory(REPLY_SEARCHRESULT, std::make_shared<RoutableFactories60::SearchResultReplyFactory>(), from6); + putRoutableFactory(REPLY_STATBUCKET, std::make_shared<RoutableFactories60::StatBucketReplyFactory>(), from6); + putRoutableFactory(REPLY_UPDATEDOCUMENT, std::make_shared<RoutableFactories60::UpdateDocumentReplyFactory>(), from6); + putRoutableFactory(REPLY_VISITORINFO, std::make_shared<RoutableFactories60::VisitorInfoReplyFactory>(), from6); + putRoutableFactory(REPLY_WRONGDISTRIBUTION, std::make_shared<RoutableFactories60::WrongDistributionReplyFactory>(), from6); } DocumentProtocol::~DocumentProtocol() = default; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp index 199d83749c2..2bd832a5855 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp @@ -2,23 +2,21 @@ #include "documentmessage.h" #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <cassert> namespace documentapi { DocumentMessage::DocumentMessage() : mbus::Message(), _priority(Priority::PRI_NORMAL_3), - _loadType(LoadType::DEFAULT), _approxSize(1024) {} +DocumentMessage::~DocumentMessage() = default; + mbus::Reply::UP DocumentMessage::createReply() const { - mbus::Reply::UP ret(doCreateReply().release()); - assert(ret.get() != nullptr); - return ret; + return doCreateReply(); } const mbus::string& diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h index 6e1507068eb..d2b8f74f716 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h @@ -2,17 +2,14 @@ #pragma once #include "documentreply.h" -#include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/messagebus/message.h> -#include <vespa/messagebus/reply.h> namespace documentapi { class DocumentMessage : public mbus::Message { private: Priority::Value _priority; - LoadType _loadType; - uint32_t _approxSize; // Not sent on wire; set by deserializer or by caller. + uint32_t _approxSize; // Not sent on wire; set by deserializer or by caller. protected: /** @@ -30,15 +27,8 @@ public: typedef std::unique_ptr<DocumentMessage> UP; typedef std::shared_ptr<DocumentMessage> SP; - /** - * Constructs a new document message with no content. - */ DocumentMessage(); - - /** - * Virtual destructor required for inheritance. - */ - virtual ~DocumentMessage() { } + ~DocumentMessage() override; /** * Creates and returns a reply to this message. This method uses the internal {@link #doCreateReply()} to @@ -47,7 +37,7 @@ public: * * @return The created reply. */ - mbus::Reply::UP createReply() const; + std::unique_ptr<mbus::Reply> createReply() const; /** * Returns the priority of this message. @@ -65,16 +55,6 @@ public: */ void setPriority(Priority::Value p) { _priority = p; }; - /** - * @return Returns the load type for this message. - */ - const LoadType& getLoadType() const { return _loadType; } - - /** - * Sets the load type for this message. - */ - void setLoadType(const LoadType& loadType) { _loadType = loadType; } - uint32_t getApproxSize() const override; void setApproxSize(uint32_t approxSize) { diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.cpp index 5a2478cbd65..d16f5d5cd66 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.cpp @@ -9,9 +9,9 @@ DocumentReply::DocumentReply(uint32_t type) : mbus::Reply(), _type(type), _priority(Priority::PRI_NORMAL_3) -{ - // empty -} +{ } + +DocumentReply::~DocumentReply() = default; const mbus::string& DocumentReply::getProtocol() const diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h index c166a4acb73..136c4983516 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h @@ -32,7 +32,7 @@ public: /** * Virtual destructor required for inheritance. */ - ~DocumentReply() { } + ~DocumentReply() override; /** * Returns the priority tag for this message. This is an optional tag added for VDS that is not interpreted by the diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetcondition.h b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetcondition.h index 33f691412dc..1af0178702d 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetcondition.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetcondition.h @@ -6,7 +6,7 @@ namespace documentapi { class TestAndSetCondition { private: - string _selection; + std::string _selection; public: TestAndSetCondition() @@ -23,7 +23,7 @@ public: TestAndSetCondition(TestAndSetCondition &&) = default; TestAndSetCondition & operator=(TestAndSetCondition &&) = default; - const string & getSelection() const { return _selection; } + const std::string & getSelection() const { return _selection; } bool isPresent() const { return !_selection.empty(); } }; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.cpp index f9a48cb0755..2c6a0d08032 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.cpp @@ -4,6 +4,7 @@ namespace documentapi { -TestAndSetMessage::~TestAndSetMessage() { } +TestAndSetMessage::TestAndSetMessage() = default; +TestAndSetMessage::~TestAndSetMessage() = default; } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.h index d3e912549c4..2e623121c85 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.h @@ -12,7 +12,8 @@ private: TestAndSetCondition _condition; public: - ~TestAndSetMessage(); + TestAndSetMessage(); + ~TestAndSetMessage() override; void setCondition(const TestAndSetCondition & condition) { _condition = condition; } const TestAndSetCondition & getCondition() const { return _condition; } }; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.cpp index baca64353fc..0683c0179cc 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.cpp @@ -1,8 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "andpolicy.h" -#include <vespa/messagebus/error.h> -#include <vespa/messagebus/errorcode.h> -#include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/routing/routingcontext.h> #include <vespa/documentapi/messagebus/documentprotocol.h> @@ -18,10 +15,7 @@ ANDPolicy::ANDPolicy(const string ¶m) } } -ANDPolicy::~ANDPolicy() -{ - // empty -} +ANDPolicy::~ANDPolicy() = default; void ANDPolicy::select(mbus::RoutingContext &context) @@ -29,11 +23,9 @@ ANDPolicy::select(mbus::RoutingContext &context) if (_hops.empty()) { context.addChildren(context.getAllRecipients()); } else { - for (std::vector<mbus::Hop>::iterator it = _hops.begin(); - it != _hops.end(); ++it) - { + for (auto & hop : _hops) { mbus::Route route = context.getRoute(); - route.setHop(0, *it); + route.setHop(0, hop); context.addChild(route); } } diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp index 82cda6c773f..21b09f32419 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp @@ -9,6 +9,7 @@ #include "asyncinitializationpolicy.h" #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/error.h> #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/vespalib/text/stringtokenizer.h> @@ -50,20 +51,23 @@ AsyncInitializationPolicy::initSynchronous() _state = State::DONE; } +namespace { + mbus::Error -AsyncInitializationPolicy::currentPolicyInitError() const -{ +currentPolicyInitError(vespalib::stringref error) { // If an init error has been recorded for the last init attempt, report // it back until we've managed to successfully complete the init step. - if (_error.empty()) { + if (error.empty()) { return mbus::Error(DocumentProtocol::ERROR_NODE_NOT_READY, "Waiting to initialize policy"); } else { return mbus::Error(DocumentProtocol::ERROR_POLICY_FAILURE, - "Error when creating policy: " + _error); + "Error when creating policy: " + error); } } +} + void AsyncInitializationPolicy::select(mbus::RoutingContext& context) { @@ -87,7 +91,7 @@ AsyncInitializationPolicy::select(mbus::RoutingContext& context) if (_state != State::DONE) { auto reply = std::make_unique<mbus::EmptyReply>(); - reply->addError(currentPolicyInitError()); + reply->addError(currentPolicyInitError(_error)); context.setReply(std::move(reply)); return; } @@ -96,7 +100,7 @@ AsyncInitializationPolicy::select(mbus::RoutingContext& context) // deadlock (executor will stall until all its tasks have finished // executing, and any queued tasks would attempt to take the mutex // we're currently holding, deadlocking both threads). - _executor.reset(nullptr); + _executor.reset(); } doSelect(context); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h index 0e30da8e7c8..3061eb3d337 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/messagebus/routing/iroutingpolicy.h> -#include <vespa/messagebus/error.h> #include <vespa/vespalib/util/executor.h> #include <vespa/documentapi/common.h> #include <map> @@ -42,8 +41,6 @@ public: void needAsynchronousInit() { _syncInit = false; } private: - mbus::Error currentPolicyInitError() const; - class Task : public vespalib::Executor::Task { public: diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.cpp index 3bf17213b26..feee7db8640 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.cpp @@ -2,6 +2,7 @@ #include "loadbalancerpolicy.h" #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/error.h> #include <vespa/messagebus/routing/ihopdirective.h> #include <vespa/messagebus/routing/routingcontext.h> #include <vespa/messagebus/routing/verbatimdirective.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp index a58f3439df2..e780806c8c0 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp @@ -3,7 +3,7 @@ #include "roundrobinpolicy.h" #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/messagebus/error.h> namespace documentapi { @@ -18,7 +18,7 @@ RoundRobinPolicy::RoundRobinPolicy(const string &) : _cache() {} -RoundRobinPolicy::~RoundRobinPolicy() {} +RoundRobinPolicy::~RoundRobinPolicy() = default; void RoundRobinPolicy::select(mbus::RoutingContext &ctx) diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index e49d412fee1..3fc1df0352a 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -4,7 +4,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/document/update/documentupdate.h> #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/messagebus/error.h> #include <vespa/documentapi/documentapi.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 7bae7dd7e77..26e609c79fa 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -22,24 +22,23 @@ RoutableFactories60::DocumentMessageFactory::encode(const mbus::Routable &obj, v { const auto &msg = static_cast<const DocumentMessage&>(obj); out.putByte(msg.getPriority()); - out.putInt(msg.getLoadType().getId()); + out.putInt(LoadType::DEFAULT.getId()); return doEncode(msg, out); } mbus::Routable::UP -RoutableFactories60::DocumentMessageFactory::decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const +RoutableFactories60::DocumentMessageFactory::decode(document::ByteBuffer &in, const LoadTypeSet&) const { uint8_t pri; in.getByte(pri); - uint32_t loadClass = decodeInt(in); + (void) decodeInt(in); DocumentMessage::UP msg = doDecode(in); if (msg) { msg->setPriority((Priority::Value)pri); - msg->setLoadType(loadTypes[loadClass]); } - return mbus::Routable::UP(msg.release()); + return msg; } bool diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h index 0a997f3ffd9..579abbda291 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h @@ -137,7 +137,7 @@ public: virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; virtual string decodeBucketSpace(document::ByteBuffer&) const; public: - CreateVisitorMessageFactory() : DocumentMessageFactory() {} + CreateVisitorMessageFactory() noexcept : DocumentMessageFactory() {} }; class CreateVisitorReplyFactory : public DocumentReplyFactory { protected: @@ -164,7 +164,7 @@ public: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; public: - DocumentListMessageFactory(const document::DocumentTypeRepo &r) + DocumentListMessageFactory(const document::DocumentTypeRepo &r) noexcept : _repo(r) {} }; class DocumentListReplyFactory : public DocumentReplyFactory { @@ -224,14 +224,14 @@ public: DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentReply &msg, vespalib::GrowableByteBuffer &buf) const override; public: - GetDocumentReplyFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + GetDocumentReplyFactory(const document::DocumentTypeRepo &r) noexcept : _repo(r) {} }; class MapVisitorMessageFactory : public DocumentMessageFactory { protected: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; public: - MapVisitorMessageFactory() : DocumentMessageFactory() {} + MapVisitorMessageFactory() noexcept : DocumentMessageFactory() {} }; class MapVisitorReplyFactory : public DocumentReplyFactory { protected: @@ -248,7 +248,7 @@ public: bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; public: void decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const; - PutDocumentMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + PutDocumentMessageFactory(const document::DocumentTypeRepo &r) noexcept : _repo(r) {} }; class PutDocumentReplyFactory : public DocumentReplyFactory { protected: @@ -275,7 +275,7 @@ public: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; public: - RemoveLocationMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + RemoveLocationMessageFactory(const document::DocumentTypeRepo &r) noexcept : _repo(r) {} }; class RemoveLocationReplyFactory : public DocumentReplyFactory { protected: @@ -324,7 +324,7 @@ public: bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; public: void decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const; - UpdateDocumentMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + UpdateDocumentMessageFactory(const document::DocumentTypeRepo &r) noexcept : _repo(r) {} }; class UpdateDocumentReplyFactory : public DocumentReplyFactory { protected: diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-GetBucketListMessage.dat Binary files differindex c42c61460ac..591e55e5849 100644 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-GetBucketListMessage.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/6.221-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/6.221-java-GetBucketListMessage.dat Binary files differindex c42c61460ac..591e55e5849 100644 --- a/documentapi/test/crosslanguagefiles/6.221-java-GetBucketListMessage.dat +++ b/documentapi/test/crosslanguagefiles/6.221-java-GetBucketListMessage.dat diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 76c76d64b79..ee8509fcf19 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -53,7 +53,7 @@ vespa_define_module( src/tests/instruction/dense_tensor_peek_function src/tests/instruction/index_lookup_table src/tests/instruction/join_with_number - src/tests/tensor/default_value_builder_factory + src/tests/streamed/value src/tests/tensor/dense_add_dimension_optimizer src/tests/tensor/dense_dimension_combiner src/tests/tensor/dense_fast_rename_optimizer @@ -71,7 +71,6 @@ vespa_define_module( src/tests/tensor/direct_sparse_tensor_builder src/tests/tensor/instruction_benchmark src/tests/tensor/onnx_wrapper - src/tests/tensor/packed_mappings src/tests/tensor/partial_add src/tests/tensor/partial_modify src/tests/tensor/partial_remove @@ -92,9 +91,9 @@ vespa_define_module( src/vespa/eval/eval/value_cache src/vespa/eval/gp src/vespa/eval/instruction + src/vespa/eval/streamed src/vespa/eval/tensor src/vespa/eval/tensor/dense - src/vespa/eval/tensor/mixed src/vespa/eval/tensor/serialization src/vespa/eval/tensor/sparse ) diff --git a/eval/src/tests/eval/engine_or_factory/engine_or_factory_override_test.cpp b/eval/src/tests/eval/engine_or_factory/engine_or_factory_override_test.cpp index 8480bb7d39e..3aa804b0722 100644 --- a/eval/src/tests/eval/engine_or_factory/engine_or_factory_override_test.cpp +++ b/eval/src/tests/eval/engine_or_factory/engine_or_factory_override_test.cpp @@ -10,16 +10,16 @@ using namespace vespalib::eval; using namespace vespalib::tensor; TEST(EngineOrFactoryOverrideTest, set_can_override_get_result) { - EngineOrFactory::set(FastValueBuilderFactory::get()); - EXPECT_EQ(EngineOrFactory::get().to_string(), "FastValueBuilderFactory"); + EngineOrFactory::set(DefaultTensorEngine::ref()); + EXPECT_EQ(EngineOrFactory::get().to_string(), "DefaultTensorEngine"); } TEST(EngineOrFactoryOverrideTest, set_with_same_value_is_allowed) { - EngineOrFactory::set(FastValueBuilderFactory::get()); + EngineOrFactory::set(DefaultTensorEngine::ref()); } TEST(EngineOrFactoryOverrideTest, set_with_another_value_is_not_allowed) { - EXPECT_THROW(EngineOrFactory::set(DefaultTensorEngine::ref()), vespalib::IllegalStateException); + EXPECT_THROW(EngineOrFactory::set(FastValueBuilderFactory::get()), vespalib::IllegalStateException); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/engine_or_factory/engine_or_factory_test.cpp b/eval/src/tests/eval/engine_or_factory/engine_or_factory_test.cpp index 6cb9cc0c89c..3acfc40f5b5 100644 --- a/eval/src/tests/eval/engine_or_factory/engine_or_factory_test.cpp +++ b/eval/src/tests/eval/engine_or_factory/engine_or_factory_test.cpp @@ -9,16 +9,16 @@ using namespace vespalib::eval; using namespace vespalib::tensor; -TEST(EngineOrFactoryTest, default_is_default_tensor_engine) { - EXPECT_EQ(EngineOrFactory::get().to_string(), "DefaultTensorEngine"); +TEST(EngineOrFactoryTest, default_is_fast_value_builder_factory) { + EXPECT_EQ(EngineOrFactory::get().to_string(), "FastValueBuilderFactory"); } TEST(EngineOrFactoryTest, set_with_same_value_is_allowed) { - EngineOrFactory::set(DefaultTensorEngine::ref()); + EngineOrFactory::set(FastValueBuilderFactory::get()); } TEST(EngineOrFactoryTest, set_with_another_value_is_not_allowed) { - EXPECT_THROW(EngineOrFactory::set(FastValueBuilderFactory::get()), vespalib::IllegalStateException); + EXPECT_THROW(EngineOrFactory::set(DefaultTensorEngine::ref()), vespalib::IllegalStateException); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 77801f44bb8..d58adbbcef0 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -8,8 +8,6 @@ using namespace vespalib::eval; -using CellType = ValueType::CellType; - const size_t npos = ValueType::Dimension::npos; ValueType type(const vespalib::string &type_str) { diff --git a/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp index aaea8fdcb28..c59d9783648 100644 --- a/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp +++ b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp @@ -8,6 +8,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_concat.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -64,63 +65,6 @@ TensorSpec perform_simpletensor_concat(const TensorSpec &a, const TensorSpec &b, return SimpleTensorEngine::ref().to_spec(*out); } -bool concat_address(const TensorSpec::Address &me, const TensorSpec::Address &other, - const std::string &concat_dim, size_t my_offset, - TensorSpec::Address &my_out, TensorSpec::Address &other_out) -{ - my_out.insert_or_assign(concat_dim, my_offset); - for (const auto &my_dim: me) { - const auto & name = my_dim.first; - const auto & label = my_dim.second; - if (name == concat_dim) { - my_out.insert_or_assign(name, label.index + my_offset); - } else { - auto pos = other.find(name); - if ((pos == other.end()) || (pos->second == label)) { - my_out.insert_or_assign(name, label); - other_out.insert_or_assign(name, label); - } else { - return false; - } - } - } - return true; -} - -bool concat_addresses(const TensorSpec::Address &a, const TensorSpec::Address &b, - const std::string &concat_dim, size_t b_offset, - TensorSpec::Address &a_out, TensorSpec::Address &b_out) -{ - return concat_address(a, b, concat_dim, 0, a_out, b_out) && - concat_address(b, a, concat_dim, b_offset, b_out, a_out); -} - -TensorSpec reference_concat(const TensorSpec &a, const TensorSpec &b, const std::string &concat_dim) { - ValueType a_type = ValueType::from_spec(a.type()); - ValueType b_type = ValueType::from_spec(b.type()); - ValueType res_type = ValueType::concat(a_type, b_type, concat_dim); - EXPECT_FALSE(res_type.is_error()); - size_t b_offset = 1; - size_t concat_dim_index = a_type.dimension_index(concat_dim); - if (concat_dim_index != ValueType::Dimension::npos) { - const auto &dim = a_type.dimensions()[concat_dim_index]; - EXPECT_TRUE(dim.is_indexed()); - b_offset = dim.size; - } - TensorSpec result(res_type.to_spec()); - for (const auto &cell_a: a.cells()) { - for (const auto &cell_b: b.cells()) { - TensorSpec::Address addr_a; - TensorSpec::Address addr_b; - if (concat_addresses(cell_a.first, cell_b.first, concat_dim, b_offset, addr_a, addr_b)) { - result.add(addr_a, cell_a.second); - result.add(addr_b, cell_b.second); - } - } - } - return result; -} - TensorSpec perform_generic_concat(const TensorSpec &a, const TensorSpec &b, const std::string &concat_dim, const ValueBuilderFactory &factory) { @@ -138,7 +82,7 @@ TEST(GenericConcatTest, generic_reference_concat_works) { const TensorSpec lhs = spec(concat_layouts[i], N()); const TensorSpec rhs = spec(concat_layouts[i + 1], Div16(N())); SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto actual = reference_concat(lhs, rhs, "y"); + auto actual = ReferenceOperations::concat(lhs, rhs, "y"); auto expect = perform_simpletensor_concat(lhs, rhs, "y"); EXPECT_EQ(actual, expect); } @@ -151,7 +95,7 @@ void test_generic_concat_with(const ValueBuilderFactory &factory) { const TensorSpec rhs = spec(concat_layouts[i + 1], Div16(N())); SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); auto actual = perform_generic_concat(lhs, rhs, "y", factory); - auto expect = reference_concat(lhs, rhs, "y"); + auto expect = ReferenceOperations::concat(lhs, rhs, "y"); EXPECT_EQ(actual, expect); } } @@ -202,7 +146,7 @@ TEST(GenericConcatTest, immediate_generic_concat_works) { const TensorSpec rhs = spec(concat_layouts[i + 1], Div16(N())); SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); auto actual = immediate_generic_concat(lhs, rhs, "y"); - auto expect = reference_concat(lhs, rhs, "y"); + auto expect = ReferenceOperations::concat(lhs, rhs, "y"); EXPECT_EQ(actual, expect); } } diff --git a/eval/src/tests/instruction/generic_create/generic_create_test.cpp b/eval/src/tests/instruction/generic_create/generic_create_test.cpp index e07db870ad2..42af4ba6621 100644 --- a/eval/src/tests/instruction/generic_create/generic_create_test.cpp +++ b/eval/src/tests/instruction/generic_create/generic_create_test.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_create.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -53,6 +54,19 @@ bool operator< (const NumberedCellSpec &a, const NumberedCellSpec &b) { return a.num < b.num; } +TensorSpec reference_create(const TensorSpec &a) { + std::vector<TensorSpec> children; + ReferenceOperations::CreateSpec spec; + for (const auto & [addr, value] : a.cells()) { + size_t child_idx = children.size(); + spec.emplace(addr, child_idx); + TensorSpec child("double"); + child.add({}, value); + children.push_back(child); + } + return ReferenceOperations::create(a.type(), spec, children); +} + TensorSpec perform_generic_create(const TensorSpec &a, const ValueBuilderFactory &factory) { ValueType res_type = ValueType::from_spec(a.type()); @@ -80,12 +94,16 @@ void test_generic_create_with(const ValueBuilderFactory &factory) { for (const auto & layout : create_layouts) { TensorSpec full = spec(layout, N()); auto actual = perform_generic_create(full, factory); - EXPECT_EQ(actual, full); + auto ref_spec = reference_create(full); + // use SimpleValue to add implicit cells with default value + auto expect = spec_from_value(*value_from_spec(ref_spec, SimpleValueBuilderFactory::get())); + EXPECT_EQ(actual, expect); for (size_t n : {2, 3, 4, 5}) { TensorSpec partial = remove_each(full, n); actual = perform_generic_create(partial, factory); - auto filled = spec_from_value(*value_from_spec(partial, SimpleValueBuilderFactory::get())); - EXPECT_EQ(actual, filled); + ref_spec = reference_create(partial); + expect = spec_from_value(*value_from_spec(ref_spec, SimpleValueBuilderFactory::get())); + EXPECT_EQ(actual, expect); } } } diff --git a/eval/src/tests/instruction/generic_join/generic_join_test.cpp b/eval/src/tests/instruction/generic_join/generic_join_test.cpp index 558f20d2e10..a81294c8d25 100644 --- a/eval/src/tests/instruction/generic_join/generic_join_test.cpp +++ b/eval/src/tests/instruction/generic_join/generic_join_test.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -53,23 +54,6 @@ bool join_address(const TensorSpec::Address &a, const TensorSpec::Address &b, Te return true; } -TensorSpec reference_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { - ValueType res_type = ValueType::join(ValueType::from_spec(a.type()), ValueType::from_spec(b.type())); - EXPECT_FALSE(res_type.is_error()); - TensorSpec result(res_type.to_spec()); - for (const auto &cell_a: a.cells()) { - for (const auto &cell_b: b.cells()) { - TensorSpec::Address addr; - if (join_address(cell_a.first, cell_b.first, addr) && - join_address(cell_b.first, cell_a.first, addr)) - { - result.add(addr, function(cell_a.second, cell_b.second)); - } - } - } - return result; -} - TensorSpec perform_generic_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function, const ValueBuilderFactory &factory) { @@ -130,7 +114,7 @@ TEST(GenericJoinTest, generic_join_works_for_simple_and_fast_values) { TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto expect = reference_join(lhs, rhs, fun); + auto expect = ReferenceOperations::join(lhs, rhs, fun); auto simple = perform_generic_join(lhs, rhs, fun, SimpleValueBuilderFactory::get()); auto fast = perform_generic_join(lhs, rhs, fun, FastValueBuilderFactory::get()); EXPECT_EQ(simple, expect); @@ -154,7 +138,7 @@ TEST(GenericJoinTest, immediate_generic_join_works) { TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto expect = reference_join(lhs, rhs, fun); + auto expect = ReferenceOperations::join(lhs, rhs, fun); auto actual = immediate_generic_join(lhs, rhs, fun); EXPECT_EQ(actual, expect); } diff --git a/eval/src/tests/instruction/generic_map/generic_map_test.cpp b/eval/src/tests/instruction/generic_map/generic_map_test.cpp index 63a9563a11b..ba6a1630777 100644 --- a/eval/src/tests/instruction/generic_map/generic_map_test.cpp +++ b/eval/src/tests/instruction/generic_map/generic_map_test.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_map.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -30,16 +31,6 @@ std::vector<Layout> map_layouts = { float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; -TensorSpec reference_map(const TensorSpec &a, map_fun_t func) { - ValueType res_type = ValueType::from_spec(a.type()); - EXPECT_FALSE(res_type.is_error()); - TensorSpec result(res_type.to_spec()); - for (const auto &cell: a.cells()) { - result.add(cell.first, func(cell.second)); - } - return result; -} - TensorSpec perform_generic_map(const TensorSpec &a, map_fun_t func, const ValueBuilderFactory &factory) { auto lhs = value_from_spec(a, factory); @@ -54,7 +45,7 @@ void test_generic_map_with(const ValueBuilderFactory &factory) { ValueType lhs_type = ValueType::from_spec(lhs.type()); for (auto func : {operation::Floor::f, operation::Fabs::f, operation::Square::f, operation::Inv::f}) { SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = reference_map(lhs, func); + auto expect = ReferenceOperations::map(lhs, func); auto actual = perform_generic_map(lhs, func, factory); EXPECT_EQ(actual, expect); } @@ -82,7 +73,7 @@ TEST(GenericMapTest, immediate_generic_map_works) { ValueType lhs_type = ValueType::from_spec(lhs.type()); for (auto func : {operation::Floor::f, operation::Fabs::f, operation::Square::f, operation::Inv::f}) { SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = reference_map(lhs, func); + auto expect = ReferenceOperations::map(lhs, func); auto actual = immediate_generic_map(lhs, func, SimpleValueBuilderFactory::get()); EXPECT_EQ(actual, expect); } diff --git a/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp b/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp index 5166ef6ccc9..a43169a6959 100644 --- a/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp +++ b/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_merge.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -33,29 +34,6 @@ std::vector<Layout> merge_layouts = { {x({"a","b","c"}),y(5)}, {x({"b","c","d"}),y(5)} }; - -TensorSpec reference_merge(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) { - ValueType res_type = ValueType::merge(ValueType::from_spec(a.type()), - ValueType::from_spec(b.type())); - EXPECT_FALSE(res_type.is_error()); - TensorSpec result(res_type.to_spec()); - for (const auto &cell: a.cells()) { - auto other = b.cells().find(cell.first); - if (other == b.cells().end()) { - result.add(cell.first, cell.second); - } else { - result.add(cell.first, fun(cell.second, other->second)); - } - } - for (const auto &cell: b.cells()) { - auto other = a.cells().find(cell.first); - if (other == a.cells().end()) { - result.add(cell.first, cell.second); - } - } - return result; -} - TensorSpec perform_generic_merge(const TensorSpec &a, const TensorSpec &b, join_fun_t fun, const ValueBuilderFactory &factory) { Stash stash; auto lhs = value_from_spec(a, factory); @@ -72,7 +50,7 @@ void test_generic_merge_with(const ValueBuilderFactory &factory) { TensorSpec rhs = spec(merge_layouts[i + 1], Div16(N())); SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f, operation::Max::f}) { - auto expect = reference_merge(lhs, rhs, fun); + auto expect = ReferenceOperations::merge(lhs, rhs, fun); auto actual = perform_generic_merge(lhs, rhs, fun, factory); EXPECT_EQ(actual, expect); } @@ -102,7 +80,7 @@ TEST(GenericMergeTest, immediate_generic_merge_works) { TensorSpec rhs = spec(merge_layouts[i + 1], Div16(N())); SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f, operation::Max::f}) { - auto expect = reference_merge(lhs, rhs, fun); + auto expect = ReferenceOperations::merge(lhs, rhs, fun); auto actual = immediate_generic_merge(lhs, rhs, fun); EXPECT_EQ(actual, expect); } diff --git a/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp b/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp index ef78d0cde68..18b1d6903dd 100644 --- a/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp +++ b/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp @@ -6,6 +6,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_peek.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/overload.h> @@ -36,55 +37,31 @@ std::vector<Layout> peek_layouts = { using PeekSpec = GenericPeek::SpecMap; -TensorSpec reference_peek(const TensorSpec ¶m, const vespalib::string &result_type, const PeekSpec &spec) { - TensorSpec result(result_type); - ValueType param_type = ValueType::from_spec(param.type()); - auto is_mapped_dim = [&](const vespalib::string &name) { - size_t dim_idx = param_type.dimension_index(name); - assert(dim_idx != ValueType::Dimension::npos); - const auto ¶m_dim = param_type.dimensions()[dim_idx]; - return param_dim.is_mapped(); - }; - TensorSpec::Address addr; +TensorSpec reference_peek(const TensorSpec ¶m, const PeekSpec &spec) { + std::vector<TensorSpec> children; + PeekSpec with_indexes; for (const auto & [dim_name, label_or_child] : spec) { + const vespalib::string &dim = dim_name; std::visit(vespalib::overload { - [&,&dim_name = dim_name](const TensorSpec::Label &label) { - addr.emplace(dim_name, label); + [&](const TensorSpec::Label &label) { + with_indexes.emplace(dim, label); }, - [&,&dim_name = dim_name](const size_t &child_value) { + [&](const size_t &child_value) { // here, label_or_child is a size_t specifying the value // we pretend a child produced - if (is_mapped_dim(dim_name)) { - // (but cast to signed first, to allow labels like the string "-2") - addr.emplace(dim_name, vespalib::make_string("%zd", ssize_t(child_value))); - } else { - addr.emplace(dim_name, child_value); - } + size_t child_idx = children.size(); + TensorSpec child("double"); + // (but cast to signed first, to allow labels like the string "-2") + child.add({}, ssize_t(child_value)); + children.push_back(child); + with_indexes.emplace(dim, child_idx); } }, label_or_child); } - for (const auto &cell: param.cells()) { - bool keep = true; - TensorSpec::Address my_addr; - for (const auto &binding: cell.first) { - auto pos = addr.find(binding.first); - if (pos == addr.end()) { - my_addr.emplace(binding.first, binding.second); - } else { - if (!(pos->second == binding.second)) { - keep = false; - } - } - } - if (keep) { - result.add(my_addr, cell.second); - } - } - return spec_from_value(*value_from_spec(result, SimpleValueBuilderFactory::get())); + return ReferenceOperations::peek(param, with_indexes, children); } - TensorSpec perform_generic_peek(const TensorSpec &a, const ValueType &result_type, PeekSpec spec, const ValueBuilderFactory &factory) { @@ -174,7 +151,9 @@ void verify_peek_equal(const TensorSpec &input, } if (reduce_dims.empty()) return; ValueType result_type = param_type.reduce(reduce_dims); - auto expect = reference_peek(input, result_type.to_spec(), spec); + auto ref_spec = reference_peek(input, spec); + // use SimpleValue to add implicit cells with default value + auto expect = spec_from_value(*value_from_spec(ref_spec, SimpleValueBuilderFactory::get())); SCOPED_TRACE(fmt("peek input: %s\n peek spec: %s\n peek result %s\n", input.to_string().c_str(), to_str(spec).c_str(), diff --git a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp index d894d273f02..fa55406be3a 100644 --- a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp +++ b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_reduce.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -34,35 +35,6 @@ std::vector<Layout> layouts = { float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; -TensorSpec reference_reduce(const TensorSpec &a, const std::vector<vespalib::string> &dims, Aggr aggr) { - Stash stash; - ValueType res_type = ValueType::from_spec(a.type()).reduce(dims); - EXPECT_FALSE(res_type.is_error()); - std::map<TensorSpec::Address,std::optional<Aggregator*>> my_map; - for (const auto &cell: a.cells()) { - TensorSpec::Address addr; - for (const auto &dim: cell.first) { - if (res_type.dimension_index(dim.first) != ValueType::Dimension::npos) { - addr.insert_or_assign(dim.first, dim.second); - } - } - auto [pos, is_empty] = my_map.emplace(addr, std::nullopt); - if (is_empty) { - pos->second = &Aggregator::create(aggr, stash); - pos->second.value()->first(cell.second); - } else { - pos->second.value()->next(cell.second); - } - } - TensorSpec result(res_type.to_spec()); - for (const auto &my_entry: my_map) { - result.add(my_entry.first, my_entry.second.value()->result()); - } - // use SimpleValue to add implicit cells with default value - const auto &factory = SimpleValueBuilderFactory::get(); - return spec_from_value(*value_from_spec(result, factory)); -} - TensorSpec perform_generic_reduce(const TensorSpec &a, const std::vector<vespalib::string> &dims, Aggr aggr, const ValueBuilderFactory &factory) { @@ -99,11 +71,14 @@ void test_generic_reduce_with(const ValueBuilderFactory &factory) { TensorSpec input = spec(layout, Div16(N())); for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX}) { for (const Domain &domain: layout) { - auto expect = reference_reduce(input, {domain.dimension}, aggr); + auto ref_spec = ReferenceOperations::reduce(input, {domain.dimension}, aggr); + // use SimpleValue to add implicit cells with default value + auto expect = spec_from_value(*value_from_spec(ref_spec, SimpleValueBuilderFactory::get())); auto actual = perform_generic_reduce(input, {domain.dimension}, aggr, factory); EXPECT_EQ(actual, expect); } - auto expect = reference_reduce(input, {}, aggr); + auto ref_spec = ReferenceOperations::reduce(input, {}, aggr); + auto expect = spec_from_value(*value_from_spec(ref_spec, SimpleValueBuilderFactory::get())); auto actual = perform_generic_reduce(input, {}, aggr, factory); EXPECT_EQ(actual, expect); } @@ -130,11 +105,13 @@ TEST(GenericReduceTest, immediate_generic_reduce_works) { TensorSpec input = spec(layout, Div16(N())); for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX}) { for (const Domain &domain: layout) { - auto expect = reference_reduce(input, {domain.dimension}, aggr); + auto ref_spec = ReferenceOperations::reduce(input, {domain.dimension}, aggr); + auto expect = spec_from_value(*value_from_spec(ref_spec, SimpleValueBuilderFactory::get())); auto actual = immediate_generic_reduce(input, {domain.dimension}, aggr); EXPECT_EQ(actual, expect); } - auto expect = reference_reduce(input, {}, aggr); + auto ref_spec = ReferenceOperations::reduce(input, {}, aggr); + auto expect = spec_from_value(*value_from_spec(ref_spec, SimpleValueBuilderFactory::get())); auto actual = immediate_generic_reduce(input, {}, aggr); EXPECT_EQ(actual, expect); } diff --git a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp index b2e30a8b78c..a7e6b8d807b 100644 --- a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp +++ b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp @@ -6,6 +6,7 @@ #include <vespa/eval/instruction/generic_rename.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/reference_operations.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -98,20 +99,6 @@ vespalib::string rename_dimension(const vespalib::string &name, const FromTo &ft return name; } -TensorSpec reference_rename(const TensorSpec &a, const FromTo &ft) { - ValueType res_type = ValueType::from_spec(a.type()).rename(ft.from, ft.to); - EXPECT_FALSE(res_type.is_error()); - TensorSpec result(res_type.to_spec()); - for (const auto &cell: a.cells()) { - TensorSpec::Address addr; - for (const auto &dim: cell.first) { - addr.insert_or_assign(rename_dimension(dim.first, ft), dim.second); - } - result.add(addr, cell.second); - } - return result; -} - TensorSpec perform_generic_rename(const TensorSpec &a, const FromTo &ft, const ValueBuilderFactory &factory) { @@ -132,7 +119,7 @@ void test_generic_rename_with(const ValueBuilderFactory &factory) { if (renamed_type.is_error()) continue; // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = reference_rename(lhs, from_to); + auto expect = ReferenceOperations::rename(lhs, from_to.from, from_to.to); auto actual = perform_generic_rename(lhs, from_to, factory); EXPECT_EQ(actual, expect); } @@ -165,7 +152,7 @@ TEST(GenericRenameTest, immediate_generic_rename_works) { if (renamed_type.is_error()) continue; // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = reference_rename(lhs, from_to); + auto expect = ReferenceOperations::rename(lhs, from_to.from, from_to.to); auto actual = immediate_generic_rename(lhs, from_to); EXPECT_EQ(actual, expect); } diff --git a/eval/src/tests/streamed/value/CMakeLists.txt b/eval/src/tests/streamed/value/CMakeLists.txt new file mode 100644 index 00000000000..d2ccced8c14 --- /dev/null +++ b/eval/src/tests/streamed/value/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_streamed_value_test_app TEST + SOURCES + streamed_value_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_streamed_value_test_app COMMAND eval_streamed_value_test_app) diff --git a/eval/src/tests/streamed/value/streamed_value_test.cpp b/eval/src/tests/streamed/value/streamed_value_test.cpp new file mode 100644 index 00000000000..3de6ba0fb63 --- /dev/null +++ b/eval/src/tests/streamed/value/streamed_value_test.cpp @@ -0,0 +1,136 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/streamed/streamed_value_builder_factory.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/instruction/generic_join.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::instruction; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +using PA = std::vector<vespalib::stringref *>; +using CPA = std::vector<const vespalib::stringref *>; + +std::vector<Layout> layouts = { + {}, + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + +std::vector<Layout> join_layouts = { + {}, {}, + {x(5)}, {x(5)}, + {x(5)}, {y(5)}, + {x(5)}, {x(5),y(5)}, + {y(3)}, {x(2),z(3)}, + {x(3),y(5)}, {y(5),z(7)}, + float_cells({x(3),y(5)}), {y(5),z(7)}, + {x(3),y(5)}, float_cells({y(5),z(7)}), + float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {y({"foo","bar","baz"})}, + {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, + {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), + float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) +}; + +TensorSpec simple_tensor_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &engine = SimpleTensorEngine::ref(); + auto lhs = engine.from_spec(a); + auto rhs = engine.from_spec(b); + const auto &result = engine.join(*lhs, *rhs, function, stash); + return engine.to_spec(result); +} + +TensorSpec streamed_value_new_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &factory = StreamedValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); + InterpretedFunction::EvalSingle single(factory, my_op); + return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs,*rhs}))); +} + +TEST(StreamedValueTest, streamed_values_can_be_converted_from_and_to_tensor_spec) { + for (const auto &layout: layouts) { + TensorSpec expect = spec(layout, N()); + std::unique_ptr<Value> value = value_from_spec(expect, StreamedValueBuilderFactory::get()); + TensorSpec actual = spec_from_value(*value); + EXPECT_EQ(actual, expect); + } +} + +TEST(StreamedValueTest, streamed_value_can_be_built_and_inspected) { + ValueType type = ValueType::from_spec("tensor<float>(x{},y[2],z{})"); + const auto &factory = StreamedValueBuilderFactory::get(); + std::unique_ptr<ValueBuilder<float>> builder = factory.create_value_builder<float>(type); + float seq = 0.0; + for (vespalib::string x: {"a", "b", "c"}) { + for (vespalib::string y: {"aa", "bb"}) { + std::vector<vespalib::stringref> addr = {x, y}; + auto subspace = builder->add_subspace(addr); + EXPECT_EQ(subspace.size(), 2); + subspace[0] = seq + 1.0; + subspace[1] = seq + 5.0; + seq += 10.0; + } + seq += 100.0; + } + std::unique_ptr<Value> value = builder->build(std::move(builder)); + EXPECT_EQ(value->index().size(), 6); + auto view = value->index().create_view({0}); + vespalib::stringref query = "b"; + vespalib::stringref label; + size_t subspace; + view->lookup(CPA{&query}); + EXPECT_TRUE(view->next_result(PA{&label}, subspace)); + EXPECT_EQ(label, "aa"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result(PA{&label}, subspace)); + EXPECT_EQ(label, "bb"); + EXPECT_EQ(subspace, 3); + EXPECT_FALSE(view->next_result(PA{&label}, subspace)); +} + +TEST(StreamedValueTest, new_generic_join_works_for_streamed_values) { + ASSERT_TRUE((join_layouts.size() % 2) == 0); + for (size_t i = 0; i < join_layouts.size(); i += 2) { + TensorSpec lhs = spec(join_layouts[i], Div16(N())); + TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Max::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = simple_tensor_join(lhs, rhs, fun); + auto actual = streamed_value_new_join(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/default_value_builder_factory/CMakeLists.txt b/eval/src/tests/tensor/default_value_builder_factory/CMakeLists.txt deleted file mode 100644 index cd7f552ec28..00000000000 --- a/eval/src/tests/tensor/default_value_builder_factory/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_default_value_builder_factory_test_app TEST - SOURCES - default_value_builder_factory_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_default_value_builder_factory_test_app COMMAND eval_default_value_builder_factory_test_app ) diff --git a/eval/src/tests/tensor/default_value_builder_factory/default_value_builder_factory_test.cpp b/eval/src/tests/tensor/default_value_builder_factory/default_value_builder_factory_test.cpp deleted file mode 100644 index bd18f3a2341..00000000000 --- a/eval/src/tests/tensor/default_value_builder_factory/default_value_builder_factory_test.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/tensor/default_value_builder_factory.h> -#include <vespa/eval/tensor/mixed/packed_mixed_tensor.h> -#include <vespa/eval/tensor/sparse/sparse_tensor.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> -#include <vespa/vespalib/gtest/gtest.h> - -using namespace vespalib; -using namespace vespalib::eval; -using namespace vespalib::tensor; -using namespace vespalib::eval::packed_mixed_tensor; - -Value::UP v_of(const TensorSpec &spec) { - return value_from_spec(spec, DefaultValueBuilderFactory::get()); -} - -using PA = std::vector<vespalib::stringref *>; -using CPA = std::vector<const vespalib::stringref *>; - -TEST(DefaultValueBuilderFactoryTest, all_built_value_types_are_correct) { - auto dbl = v_of(TensorSpec("double").add({}, 3.0)); - auto trivial = v_of(TensorSpec("tensor(x[1])").add({{"x",0}}, 7.0)); - auto dense = v_of(TensorSpec("tensor<float>(x[2],y[3])").add({{"x",1},{"y",2}}, 17.0)); - auto sparse = v_of(TensorSpec("tensor(x{},y{})").add({{"x","foo"},{"y","bar"}}, 31.0)); - auto mixed = v_of(TensorSpec("tensor<float>(x[2],y{})").add({{"x",1},{"y","quux"}}, 42.0)); - - EXPECT_TRUE(dynamic_cast<DoubleValue *>(dbl.get())); - EXPECT_TRUE(dynamic_cast<DenseTensorView *>(trivial.get())); - EXPECT_TRUE(dynamic_cast<DenseTensorView *>(dense.get())); - EXPECT_TRUE(dynamic_cast<SparseTensor *>(sparse.get())); - EXPECT_TRUE(dynamic_cast<PackedMixedTensor *>(mixed.get())); - - EXPECT_EQ(dbl->as_double(), 3.0); - EXPECT_EQ(trivial->cells().typify<double>()[0], 7.0); - EXPECT_EQ(dense->cells().typify<float>()[5], 17.0); - EXPECT_EQ(sparse->cells().typify<double>()[0], 31.0); - EXPECT_EQ(mixed->cells().typify<float>()[1], 42.0); - - stringref y_look = "bar"; - stringref x_res = "xxx"; - auto view = sparse->index().create_view({1}); - view->lookup(CPA{&y_look}); - size_t ss = 12345; - bool br = view->next_result(PA{&x_res}, ss); - EXPECT_TRUE(br); - EXPECT_EQ(ss, 0); - EXPECT_EQ(x_res, "foo"); - br = view->next_result(PA{&x_res}, ss); - EXPECT_FALSE(br); - - ss = 12345; - view = mixed->index().create_view({}); - view->lookup({}); - br = view->next_result(PA{&x_res}, ss); - EXPECT_TRUE(br); - EXPECT_EQ(ss, 0); - EXPECT_EQ(x_res, "quux"); -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp index d9d4c221164..bcee6471f76 100644 --- a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp @@ -8,6 +8,7 @@ using namespace vespalib::tensor; using namespace vespalib::tensor::sparse; using vespalib::eval::TensorSpec; +using vespalib::eval::CellType; using vespalib::eval::ValueType; void @@ -36,7 +37,7 @@ assertCellValue(double expValue, const TensorAddress &address, bool found = tensor.index().lookup_address(addressRef, idx); EXPECT_TRUE(found); auto cells = tensor.cells(); - if (EXPECT_TRUE(cells.type == ValueType::CellType::DOUBLE)) { + if (EXPECT_TRUE(cells.type == CellType::DOUBLE)) { auto arr = cells.typify<double>(); EXPECT_EQUAL(expValue, arr[idx]); } diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 816923bb87c..f11678d3ee7 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -36,7 +36,6 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/optimize_tensor_function.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/default_value_builder_factory.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/objects/nbostream.h> @@ -95,7 +94,7 @@ template <typename ...Ds> void add_cells(TensorSpec &spec, double &seq, TensorSp } template <typename ...Ds> TensorSpec make_spec(double seq, const Ds &...ds) { - TensorSpec spec(ValueType::tensor_type({ds...}, ValueType::CellType::FLOAT).to_spec()); + TensorSpec spec(ValueType::tensor_type({ds...}, CellType::FLOAT).to_spec()); add_cells(spec, seq, TensorSpec::Address(), ds...); return spec; } @@ -354,6 +353,7 @@ MyParam::~MyParam() = default; struct EvalOp { using UP = std::unique_ptr<EvalOp>; + Stash my_stash; const Impl &impl; MyParam my_param; std::vector<Value::UP> values; @@ -361,8 +361,8 @@ struct EvalOp { EvalSingle single; EvalOp(const EvalOp &) = delete; EvalOp &operator=(const EvalOp &) = delete; - EvalOp(Instruction op, const std::vector<CREF<TensorSpec>> &stack_spec, const Impl &impl_in) - : impl(impl_in), my_param(), values(), stack(), single(impl.engine, op) + EvalOp(Stash &&stash_in, Instruction op, const std::vector<CREF<TensorSpec>> &stack_spec, const Impl &impl_in) + : my_stash(std::move(stash_in)), impl(impl_in), my_param(), values(), stack(), single(impl.engine, op) { for (const TensorSpec &spec: stack_spec) { values.push_back(impl.create_value(spec)); @@ -371,14 +371,51 @@ struct EvalOp { stack.push_back(*value.get()); } } - EvalOp(Instruction op, const TensorSpec &p0, const Impl &impl_in) - : impl(impl_in), my_param(p0, impl), values(), stack(), single(impl.engine, op, my_param) + EvalOp(Stash &&stash_in, Instruction op, const TensorSpec &p0, const Impl &impl_in) + : my_stash(std::move(stash_in)), impl(impl_in), my_param(p0, impl), values(), stack(), single(impl.engine, op, my_param) { } TensorSpec result() { return impl.create_spec(single.eval(stack)); } - double estimate_cost_us() { - auto actual = [&](){ single.eval(stack); }; - return BenchmarkTimer::benchmark(actual, budget) * 1000.0 * 1000.0; + size_t suggest_loop_cnt() { + size_t loop_cnt = 1; + auto my_loop = [&](){ + for (size_t i = 0; i < loop_cnt; ++i) { + single.eval(stack); + } + }; + for (;;) { + vespalib::BenchmarkTimer timer(0.0); + for (size_t i = 0; i < 5; ++i) { + timer.before(); + my_loop(); + timer.after(); + } + double min_time = timer.min_time(); + if (min_time > 0.004) { + break; + } else { + loop_cnt *= 2; + } + } + return std::max(loop_cnt, size_t(8)); + } + double estimate_cost_us(size_t self_loop_cnt, size_t ref_loop_cnt) { + size_t loop_cnt = ((self_loop_cnt * 128) < ref_loop_cnt) ? self_loop_cnt : ref_loop_cnt; + assert((loop_cnt % 8) == 0); + auto my_loop = [&](){ + for (size_t i = 0; (i + 7) < loop_cnt; i += 8) { + for (size_t j = 0; j < 8; ++j) { + single.eval(stack); + } + } + }; + BenchmarkTimer timer(budget); + while (timer.has_budget()) { + timer.before(); + my_loop(); + timer.after(); + } + return timer.min_time() * 1000.0 * 1000.0 / double(loop_cnt); } }; @@ -396,8 +433,12 @@ void benchmark(const vespalib::string &desc, const std::vector<EvalOp::UP> &list } } BenchmarkResult result(desc, list.size()); + std::vector<size_t> loop_cnt(list.size()); for (const auto &eval: list) { - double time = eval->estimate_cost_us(); + loop_cnt[eval->impl.order] = eval->suggest_loop_cnt(); + } + for (const auto &eval: list) { + double time = eval->estimate_cost_us(loop_cnt[eval->impl.order], loop_cnt[1]); result.sample(eval->impl.order, time); fprintf(stderr, " %s(%s): %10.3f us\n", eval->impl.name.c_str(), eval->impl.short_name.c_str(), time); } @@ -420,9 +461,10 @@ void benchmark_join(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_join(lhs_type, rhs_type, function, stash); + Stash my_stash; + auto op = impl.create_join(lhs_type, rhs_type, function, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -439,9 +481,10 @@ void benchmark_reduce(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_reduce(lhs_type, aggr, dims, stash); + Stash my_stash; + auto op = impl.create_reduce(lhs_type, aggr, dims, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -459,9 +502,10 @@ void benchmark_rename(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_rename(lhs_type, from, to, stash); + Stash my_stash; + auto op = impl.create_rename(lhs_type, from, to, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -480,9 +524,10 @@ void benchmark_merge(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_merge(lhs_type, rhs_type, function, stash); + Stash my_stash; + auto op = impl.create_merge(lhs_type, rhs_type, function, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -496,9 +541,10 @@ void benchmark_map(const vespalib::string &desc, const TensorSpec &lhs, operatio ASSERT_FALSE(lhs_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_map(lhs_type, function, stash); + Stash my_stash; + auto op = impl.create_map(lhs_type, function, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -517,9 +563,10 @@ void benchmark_concat(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_concat(lhs_type, rhs_type, dimension, stash); + Stash my_stash; + auto op = impl.create_concat(lhs_type, rhs_type, dimension, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -536,10 +583,11 @@ void benchmark_tensor_create(const vespalib::string &desc, const TensorSpec &pro } std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_tensor_create(proto_type, proto, stash); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + Stash my_stash; + auto op = impl.create_tensor_create(proto_type, proto, my_stash); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } - benchmark(desc, list); + benchmark(desc, list); } //----------------------------------------------------------------------------- @@ -550,8 +598,9 @@ void benchmark_tensor_lambda(const vespalib::string &desc, const ValueType &type ASSERT_FALSE(p0_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_tensor_lambda(type, function, p0_type, stash); - list.push_back(std::make_unique<EvalOp>(op, p0, impl)); + Stash my_stash; + auto op = impl.create_tensor_lambda(type, function, p0_type, my_stash); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, p0, impl)); } benchmark(desc, list); } @@ -571,8 +620,9 @@ void benchmark_tensor_peek(const vespalib::string &desc, const TensorSpec &lhs, } std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_tensor_peek(type, peek_spec, stash); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + Stash my_stash; + auto op = impl.create_tensor_peek(type, peek_spec, my_stash); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -610,11 +660,11 @@ void benchmark_encode_decode(const vespalib::string &desc, const TensorSpec &pro BenchmarkResult encode_result(desc + " <encode>", impl_list.size()); BenchmarkResult decode_result(desc + " <decode>", impl_list.size()); for (const Impl &impl: impl_list) { - constexpr size_t loop_cnt = 16; + constexpr size_t loop_cnt = 32; auto value = impl.create_value(proto); BenchmarkTimer encode_timer(2 * budget); BenchmarkTimer decode_timer(2 * budget); - while (encode_timer.has_budget() || decode_timer.has_budget()) { + while (encode_timer.has_budget()) { std::array<vespalib::nbostream, loop_cnt> data; std::array<Value::UP, loop_cnt> object; encode_timer.before(); diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp index fce7ccc6411..efab0571e62 100644 --- a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp +++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp @@ -163,7 +163,7 @@ TEST(OnnxTest, simple_onnx_model_can_be_evaluated) ctx.bind_param(2, bias); ctx.eval(); auto cells = output.cells(); - EXPECT_EQ(cells.type, ValueType::CellType::FLOAT); + EXPECT_EQ(cells.type, CellType::FLOAT); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); //------------------------------------------------------------------------- @@ -209,7 +209,7 @@ TEST(OnnxTest, dynamic_onnx_model_can_be_evaluated) ctx.bind_param(2, bias); ctx.eval(); auto cells = output.cells(); - EXPECT_EQ(cells.type, ValueType::CellType::FLOAT); + EXPECT_EQ(cells.type, CellType::FLOAT); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); //------------------------------------------------------------------------- @@ -255,7 +255,7 @@ TEST(OnnxTest, int_types_onnx_model_can_be_evaluated) ctx.bind_param(2, bias); ctx.eval(); auto cells = output.cells(); - EXPECT_EQ(cells.type, ValueType::CellType::DOUBLE); + EXPECT_EQ(cells.type, CellType::DOUBLE); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); //------------------------------------------------------------------------- diff --git a/eval/src/tests/tensor/packed_mappings/CMakeLists.txt b/eval/src/tests/tensor/packed_mappings/CMakeLists.txt deleted file mode 100644 index 2d11755a0c5..00000000000 --- a/eval/src/tests/tensor/packed_mappings/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_executable(eval_packed_mappings_test_app TEST - SOURCES - packed_mappings_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_packed_mappings_test_app COMMAND eval_packed_mappings_test_app) - -vespa_add_executable(eval_packed_mixed_test_app TEST - SOURCES - packed_mixed_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_packed_mixed_test_app COMMAND eval_packed_mixed_test_app) diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp deleted file mode 100644 index be25dd7f444..00000000000 --- a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/tensor/mixed/packed_labels.h> -#include <vespa/eval/tensor/mixed/packed_mappings.h> -#include <vespa/eval/tensor/mixed/packed_mappings_builder.h> -#include <vespa/eval/tensor/mixed/packed_mixed_tensor.h> -#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h> -#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> -#include <vespa/vespalib/gtest/gtest.h> -#include <stdlib.h> -#include <assert.h> -#include <set> - -using namespace vespalib::eval; -using namespace vespalib::eval::packed_mixed_tensor; - -namespace { - -uint32_t random_range(uint32_t from, uint32_t to) { - assert(from + 1 < to); - int unif = rand() % (to - from); - return from + unif; -} - -const char *mixed_tensor_types[] = { - "tensor<float>(x{})", - "tensor<float>(a{},b{},c{},d{},e{},f{})", - "tensor<float>(x{},y{})", - "tensor<float>(x{},z[3])", - "tensor<float>(w[5],x{},y{},z[3])" -}; - -const char *float_tensor_types[] = { - "tensor<float>(x{})", - "tensor<float>(x{},y{})", - "tensor<float>(x{},z[3])", - "tensor<float>(w[5],x{},y{},z[3])", - "tensor<float>(z[2])" -}; - - vespalib::string label1(""), - label2("foo"), - label3("bar"); - vespalib::string label4("foobar"), - label5("barfoo"), - label6("other"); - vespalib::string label7("long text number one"), - label8("long text number two"), - label9("long text number three"); - -std::vector<vespalib::stringref> -generate_random_address(uint32_t dims) -{ - std::vector<vespalib::stringref> foo(dims, label1); - for (auto & ref : foo) { - size_t pct = random_range(0, 100); - if (pct < 5) { ref = label1; } - else if (pct < 30) { ref = label2; } - else if (pct < 55) { ref = label3; } - else if (pct < 65) { ref = label4; } - else if (pct < 75) { ref = label5; } - else if (pct < 85) { ref = label6; } - else if (pct < 90) { ref = label7; } - else if (pct < 95) { ref = label8; } - else { ref = label9; } - } - return foo; -} - -} // namespace <unnamed> - -class MappingsBuilderTest : public ::testing::Test { -public: - std::unique_ptr<PackedMappingsBuilder> builder; - std::unique_ptr<PackedMappings> built; - - MappingsBuilderTest() = default; - - virtual ~MappingsBuilderTest() = default; - - void build_and_compare() { - ASSERT_TRUE(builder); - built = builder->build_mappings(); - ASSERT_TRUE(built); - EXPECT_EQ(builder->num_mapped_dims(), built->num_mapped_dims()); - EXPECT_EQ(builder->size(), built->size()); - for (size_t idx = 0; idx < built->size(); ++idx) { - std::vector<vespalib::stringref> got(builder->num_mapped_dims()); - built->fill_address_by_sortid(idx, got); - printf("Got address:"); - for (auto ref : got) { - printf(" '%s'", ref.data()); - } - uint32_t subspace = built->subspace_of_address(got); - uint32_t original = builder->add_mapping_for(got); - printf(" -> %u\n", original); - EXPECT_EQ(subspace, original); - } - } -}; - -TEST_F(MappingsBuilderTest, empty_mapping) -{ - for (uint32_t dims : { 0, 1, 2, 3 }) { - builder = std::make_unique<PackedMappingsBuilder>(dims); - build_and_compare(); - } -} - -TEST_F(MappingsBuilderTest, just_one) -{ - vespalib::string label("foobar"); - for (uint32_t dims : { 0, 1, 2, 3, 7 }) { - builder = std::make_unique<PackedMappingsBuilder>(dims); - std::vector<vespalib::stringref> foo(dims, label); - uint32_t idx = builder->add_mapping_for(foo); - EXPECT_EQ(idx, 0); - build_and_compare(); - } -} - -TEST_F(MappingsBuilderTest, some_random) -{ - for (uint32_t dims : { 1, 2, 5 }) { - builder = std::make_unique<PackedMappingsBuilder>(dims); - uint32_t cnt = random_range(dims*5, dims*20); - printf("Generate %u addresses for %u dims\n", cnt, dims); - for (uint32_t i = 0; i < cnt; ++i) { - auto foo = generate_random_address(dims); - uint32_t idx = builder->add_mapping_for(foo); - EXPECT_LE(idx, i); - } - build_and_compare(); - } -} - -class MixedBuilderTest : public ::testing::Test { -public: - std::unique_ptr<PackedMixedTensorBuilder<float>> builder; - std::unique_ptr<Value> built; - - MixedBuilderTest() = default; - - virtual ~MixedBuilderTest() = default; - - size_t expected_value = 0; - - void build_and_compare(size_t expect_size) { - built.reset(nullptr); - EXPECT_FALSE(built); - ASSERT_TRUE(builder); - built = builder->build(std::move(builder)); - EXPECT_FALSE(builder); - ASSERT_TRUE(built); - EXPECT_EQ(built->index().size(), expect_size); - auto cells = built->cells().typify<float>(); - for (float f : cells) { - float expect = ++expected_value; - EXPECT_EQ(f, expect); - } - } -}; - -TEST_F(MixedBuilderTest, empty_mapping) -{ - for (auto type_spec : mixed_tensor_types) { - ValueType type = ValueType::from_spec(type_spec); - size_t dims = type.count_mapped_dimensions(); - size_t dsss = type.dense_subspace_size(); - EXPECT_GT(dims, 0); - EXPECT_GT(dsss, 0); - builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); - build_and_compare(0); - } -} - -TEST_F(MixedBuilderTest, just_one) -{ - size_t counter = 0; - for (auto type_spec : float_tensor_types) { - ValueType type = ValueType::from_spec(type_spec); - size_t dims = type.count_mapped_dimensions(); - size_t dsss = type.dense_subspace_size(); - EXPECT_GT(dsss, 0); - builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); - auto address = generate_random_address(dims); - auto ref = builder->add_subspace(address); - EXPECT_EQ(ref.size(), dsss); - for (size_t i = 0; i < ref.size(); ++i) { - ref[i] = ++counter; - } - build_and_compare(1); - } -} - -TEST_F(MixedBuilderTest, some_random) -{ - size_t counter = 0; - for (auto type_spec : mixed_tensor_types) { - ValueType type = ValueType::from_spec(type_spec); - uint32_t dims = type.count_mapped_dimensions(); - uint32_t dsss = type.dense_subspace_size(); - EXPECT_GT(dims, 0); - EXPECT_GT(dsss, 0); - builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); - - uint32_t cnt = random_range(dims*5, dims*20); - printf("MixBuild: generate %u addresses for %u dims\n", cnt, dims); - std::set<std::vector<vespalib::stringref>> seen; - for (uint32_t i = 0; i < cnt; ++i) { - auto address = generate_random_address(dims); - if (seen.insert(address).second) { - auto ref = builder->add_subspace(address); - EXPECT_EQ(ref.size(), dsss); - for (size_t j = 0; j < ref.size(); ++j) { - ref[j] = ++counter; - } - } - } - printf("MixBuild: generated %zu unique addresses\n", seen.size()); - build_and_compare(seen.size()); - } -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp deleted file mode 100644 index e8c57384335..00000000000 --- a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/eval/test/tensor_model.hpp> -#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> -#include <vespa/vespalib/gtest/gtest.h> - -using namespace vespalib::eval; -using namespace vespalib::eval::test; - -using PA = std::vector<vespalib::stringref *>; -using CPA = std::vector<const vespalib::stringref *>; - -std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) -}; - -TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_spec) { - for (const auto &layout: layouts) { - TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, PackedMixedTensorBuilderFactory::get()); - TensorSpec actual = spec_from_value(*value); - EXPECT_EQ(actual, expect); - } -} - -TEST(PackedMixedTest, packed_mixed_tensors_can_be_built_and_inspected) { - ValueType type = ValueType::from_spec("tensor<float>(x{},y[2],z{})"); - const auto & factory = PackedMixedTensorBuilderFactory::get(); - std::unique_ptr<ValueBuilder<float>> builder = factory.create_value_builder<float>(type); - float seq = 0.0; - for (vespalib::string x: {"a", "b", "c"}) { - for (vespalib::string y: {"aa", "bb"}) { - std::vector<vespalib::stringref> addr = {x, y}; - auto subspace = builder->add_subspace(addr); - EXPECT_EQ(subspace.size(), 2); - subspace[0] = seq + 1.0; - subspace[1] = seq + 5.0; - seq += 10.0; - } - seq += 100.0; - } - std::unique_ptr<Value> value = builder->build(std::move(builder)); - EXPECT_EQ(value->index().size(), 6); - auto view = value->index().create_view({0}); - vespalib::stringref query = "b"; - vespalib::stringref label; - size_t subspace; - view->lookup(CPA{&query}); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "aa"); - EXPECT_EQ(subspace, 2); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "bb"); - EXPECT_EQ(subspace, 3); - EXPECT_FALSE(view->next_result(PA{&label}, subspace)); - - query = "c"; - view->lookup(CPA{&query}); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "aa"); - EXPECT_EQ(subspace, 4); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "bb"); - EXPECT_EQ(subspace, 5); - EXPECT_FALSE(view->next_result(PA{&label}, subspace)); - - query = "notpresent"; - view->lookup(CPA{&query}); - EXPECT_FALSE(view->next_result(PA{&label}, subspace)); - - view = value->index().create_view({1}); - query = "aa"; - view->lookup(CPA{&query}); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "a"); - EXPECT_EQ(subspace, 0); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "b"); - EXPECT_EQ(subspace, 2); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "c"); - EXPECT_EQ(subspace, 4); - EXPECT_FALSE(view->next_result(PA{&label}, subspace)); - - query = "bb"; - view->lookup(CPA{&query}); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "a"); - EXPECT_EQ(subspace, 1); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "b"); - EXPECT_EQ(subspace, 3); - EXPECT_TRUE(view->next_result(PA{&label}, subspace)); - EXPECT_EQ(label, "c"); - EXPECT_EQ(subspace, 5); - EXPECT_FALSE(view->next_result(PA{&label}, subspace)); - - query = "notpresent"; - view->lookup(CPA{&query}); - EXPECT_FALSE(view->next_result(PA{&label}, subspace)); - - view = value->index().create_view({0,1}); - vespalib::stringref query_x = "b"; - vespalib::stringref query_y = "bb"; - CPA addr = {&query_x, &query_y}; - view->lookup(addr); - EXPECT_TRUE(view->next_result({}, subspace)); - EXPECT_EQ(subspace, 3); - EXPECT_FALSE(view->next_result({}, subspace)); - - view = value->index().create_view({}); - vespalib::stringref label_x; - vespalib::stringref label_y; - view->lookup({}); - - const std::vector<vespalib::stringref*> out({&label_x, &label_y}); - EXPECT_TRUE(view->next_result(out, subspace)); - EXPECT_EQ(label_x, "a"); - EXPECT_EQ(label_y, "aa"); - EXPECT_EQ(subspace, 0); - EXPECT_TRUE(view->next_result(out, subspace)); - EXPECT_EQ(label_x, "a"); - EXPECT_EQ(label_y, "bb"); - EXPECT_EQ(subspace, 1); - EXPECT_TRUE(view->next_result(out, subspace)); - EXPECT_EQ(label_x, "b"); - EXPECT_EQ(label_y, "aa"); - EXPECT_EQ(subspace, 2); - EXPECT_TRUE(view->next_result(out, subspace)); - EXPECT_EQ(label_x, "b"); - EXPECT_EQ(label_y, "bb"); - EXPECT_EQ(subspace, 3); - EXPECT_TRUE(view->next_result(out, subspace)); - EXPECT_EQ(label_x, "c"); - EXPECT_EQ(label_y, "aa"); - EXPECT_EQ(subspace, 4); - EXPECT_TRUE(view->next_result(out, subspace)); - EXPECT_EQ(label_x, "c"); - EXPECT_EQ(label_y, "bb"); - EXPECT_EQ(subspace, 5); - EXPECT_FALSE(view->next_result(out, subspace)); -} - - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp index e182fffa890..5af2396f5ec 100644 --- a/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp +++ b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp @@ -124,20 +124,31 @@ expect_partial_remove(const TensorSpec& input, const TensorSpec& remove, const T } TEST(PartialRemoveTest, remove_where_address_is_not_fully_specified) { - auto input = TensorSpec("tensor(x{},y{})"). + auto input_sparse = TensorSpec("tensor(x{},y{})"). add({{"x", "a"},{"y", "c"}}, 3.0). add({{"x", "a"},{"y", "d"}}, 5.0). add({{"x", "b"},{"y", "c"}}, 7.0); - expect_partial_remove(input,TensorSpec("tensor(x{})").add({{"x", "a"}}, 1.0), + expect_partial_remove(input_sparse, TensorSpec("tensor(x{})").add({{"x", "a"}}, 1.0), TensorSpec("tensor(x{},y{})").add({{"x", "b"},{"y", "c"}}, 7.0)); - expect_partial_remove(input, TensorSpec("tensor(y{})").add({{"y", "c"}}, 1.0), + expect_partial_remove(input_sparse, TensorSpec("tensor(y{})").add({{"y", "c"}}, 1.0), TensorSpec("tensor(x{},y{})").add({{"x", "a"},{"y", "d"}}, 5.0)); - expect_partial_remove(input, TensorSpec("tensor(y{})").add({{"y", "d"}}, 1.0), + expect_partial_remove(input_sparse, TensorSpec("tensor(y{})").add({{"y", "d"}}, 1.0), TensorSpec("tensor(x{},y{})").add({{"x", "a"},{"y", "c"}}, 3.0) .add({{"x", "b"},{"y", "c"}}, 7.0)); + + auto input_mixed = TensorSpec("tensor(x{},y{},z[1])"). + add({{"x", "a"},{"y", "c"},{"z", 0}}, 3.0). + add({{"x", "a"},{"y", "d"},{"z", 0}}, 5.0). + add({{"x", "b"},{"y", "c"},{"z", 0}}, 7.0); + + expect_partial_remove(input_mixed,TensorSpec("tensor(x{})").add({{"x", "a"}}, 1.0), + TensorSpec("tensor(x{},y{},z[1])").add({{"x", "b"},{"y", "c"},{"z", 0}}, 7.0)); + + expect_partial_remove(input_mixed, TensorSpec("tensor(y{})").add({{"y", "c"}}, 1.0), + TensorSpec("tensor(x{},y{},z[1])").add({{"x", "a"},{"y", "d"},{"z", 0}}, 5.0)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp index 233aff0e425..6468f50a00e 100644 --- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp +++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp @@ -3,11 +3,13 @@ #include <vespa/eval/eval/test/tensor_conformance.h> #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/streamed/streamed_value_builder_factory.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/vespalib/util/stringfmt.h> using vespalib::eval::SimpleValueBuilderFactory; +using vespalib::eval::StreamedValueBuilderFactory; using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::SimpleTensorEngine; using vespalib::eval::test::TensorConformance; @@ -29,6 +31,10 @@ TEST("require that SimpleValue implementation passes all conformance tests") { TEST_DO(TensorConformance::run_tests(module_src_path, SimpleValueBuilderFactory::get())); } +TEST("require that StreamedValue implementation passes all conformance tests") { + TEST_DO(TensorConformance::run_tests(module_src_path, StreamedValueBuilderFactory::get())); +} + TEST("require that FastValue implementation passes all conformance tests") { TEST_DO(TensorConformance::run_tests(module_src_path, FastValueBuilderFactory::get())); } diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt index ee9a793bba0..952640195b1 100644 --- a/eval/src/vespa/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/CMakeLists.txt @@ -7,9 +7,9 @@ vespa_add_library(vespaeval $<TARGET_OBJECTS:eval_eval_test> $<TARGET_OBJECTS:eval_eval_value_cache> $<TARGET_OBJECTS:eval_gp> + $<TARGET_OBJECTS:eval_streamed> $<TARGET_OBJECTS:eval_tensor> $<TARGET_OBJECTS:eval_tensor_dense> - $<TARGET_OBJECTS:eval_tensor_mixed> $<TARGET_OBJECTS:eval_tensor_serialization> $<TARGET_OBJECTS:eval_tensor_sparse> INSTALL lib64 diff --git a/eval/src/vespa/eval/eval/cell_type.cpp b/eval/src/vespa/eval/eval/cell_type.cpp index e5729c547b0..365a3f59a56 100644 --- a/eval/src/vespa/eval/eval/cell_type.cpp +++ b/eval/src/vespa/eval/eval/cell_type.cpp @@ -1,3 +1,19 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "cell_type.h" +#include <stdio.h> +#include <cstdlib> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> + +using vespalib::make_string_short::fmt; + +namespace vespalib::eval { + +void +CellTypeUtils::bad_argument(uint32_t id) +{ + throw IllegalArgumentException(fmt("Unknown CellType id=%u", id)); +} + +} diff --git a/eval/src/vespa/eval/eval/cell_type.h b/eval/src/vespa/eval/eval/cell_type.h index 0e878f26f47..49114d04bfe 100644 --- a/eval/src/vespa/eval/eval/cell_type.h +++ b/eval/src/vespa/eval/eval/cell_type.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/vespalib/util/typify.h> -#include <cstdlib> +#include <cstdint> namespace vespalib::eval { @@ -25,6 +25,26 @@ template <typename CT> inline CellType get_cell_type(); template <> inline CellType get_cell_type<double>() { return CellType::DOUBLE; } template <> inline CellType get_cell_type<float>() { return CellType::FLOAT; } +struct CellTypeUtils { + static void bad_argument [[ noreturn ]] (uint32_t id); + + static constexpr uint32_t alignment(CellType cell_type) { + switch (cell_type) { + case CellType::DOUBLE: return sizeof(double); + case CellType::FLOAT: return sizeof(float); + } + bad_argument((uint32_t)cell_type); + } + + static constexpr size_t mem_size(CellType cell_type, size_t sz) { + switch (cell_type) { + case CellType::DOUBLE: return sz * sizeof(double); + case CellType::FLOAT: return sz * sizeof(float); + } + bad_argument((uint32_t)cell_type); + } +}; + struct TypifyCellType { template <typename T> using Result = TypifyResultType<T>; template <typename F> static decltype(auto) resolve(CellType value, F &&f) { @@ -32,7 +52,7 @@ struct TypifyCellType { case CellType::DOUBLE: return f(Result<double>()); case CellType::FLOAT: return f(Result<float>()); } - abort(); + CellTypeUtils::bad_argument((uint32_t)value); } }; diff --git a/eval/src/vespa/eval/eval/engine_or_factory.cpp b/eval/src/vespa/eval/eval/engine_or_factory.cpp index 4a95a57e10e..36251820c23 100644 --- a/eval/src/vespa/eval/eval/engine_or_factory.cpp +++ b/eval/src/vespa/eval/eval/engine_or_factory.cpp @@ -12,8 +12,6 @@ #include <vespa/eval/instruction/generic_reduce.h> #include <vespa/eval/instruction/generic_rename.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/default_value_builder_factory.h> -#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> #include <vespa/vespalib/data/memory.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> @@ -26,7 +24,7 @@ using namespace vespalib::eval::instruction; namespace vespalib::eval { -EngineOrFactory EngineOrFactory::_default{tensor::DefaultTensorEngine::ref()}; +EngineOrFactory EngineOrFactory::_default{FastValueBuilderFactory::get()}; EngineOrFactory @@ -173,12 +171,6 @@ EngineOrFactory::to_string() const if (&factory() == &SimpleValueBuilderFactory::get()) { return "SimpleValueBuilderFactory"; } - if (&factory() == &tensor::DefaultValueBuilderFactory::get()) { - return "DefaultValueBuilderFactory"; - } - if (&factory() == &PackedMixedTensorBuilderFactory::get()) { - return "PackedMixedTensorBuilderFactory"; - } } return "???"; } diff --git a/eval/src/vespa/eval/eval/fast_sparse_map.cpp b/eval/src/vespa/eval/eval/fast_sparse_map.cpp index 2e95934286c..e5ffbb5c515 100644 --- a/eval/src/vespa/eval/eval/fast_sparse_map.cpp +++ b/eval/src/vespa/eval/eval/fast_sparse_map.cpp @@ -7,6 +7,12 @@ namespace vespalib::eval { FastSparseMap::~FastSparseMap() = default; +FastSparseMap& +FastSparseMap::operator=(const FastSparseMap& rhs) = default; + +FastSparseMap& +FastSparseMap::operator=(FastSparseMap&& rhs) = default; + const FastSparseMap::HashedLabel FastSparseMap::empty_label; } diff --git a/eval/src/vespa/eval/eval/fast_sparse_map.h b/eval/src/vespa/eval/eval/fast_sparse_map.h index 0d7597a19a0..99e01e8c823 100644 --- a/eval/src/vespa/eval/eval/fast_sparse_map.h +++ b/eval/src/vespa/eval/eval/fast_sparse_map.h @@ -97,6 +97,9 @@ public: } ~FastSparseMap(); + FastSparseMap& operator=(const FastSparseMap& rhs); + FastSparseMap& operator=(FastSparseMap&& rhs); + MemoryUsage estimate_extra_memory_usage() const { MemoryUsage extra_usage; size_t map_self_size = sizeof(_map); diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index ff94f94efbc..9914378cc9e 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -390,20 +390,12 @@ FastValueIndex::sparse_only_merge(const ValueType &res_type, const Fun &fun, const FastValueIndex &lhs, const FastValueIndex &rhs, ConstArrayRef<LCT> lhs_cells, ConstArrayRef<RCT> rhs_cells, Stash &stash) { - auto &result = stash.create<FastValue<OCT>>(res_type, lhs.map.num_dims(), 1, lhs.map.size()+rhs.map.size()); - lhs.map.each_map_entry([&](auto lhs_subspace, auto hash) - { - auto idx = result.my_index.map.add_mapping(lhs.map.make_addr(lhs_subspace), hash); - if (__builtin_expect((idx == result.my_cells.size), true)) { - auto rhs_subspace = rhs.map.lookup(hash); - if (rhs_subspace != FastSparseMap::npos()) { - auto cell_value = fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace]); - result.my_cells.push_back_fast(cell_value); - } else { - result.my_cells.push_back_fast(lhs_cells[lhs_subspace]); - } - } - }); + size_t guess_size = lhs.map.size() + rhs.map.size(); + auto &result = stash.create<FastValue<OCT>>(res_type, lhs.map.num_dims(), 1, guess_size); + result.my_index = lhs; + for (auto val : lhs_cells) { + result.my_cells.push_back_fast(val); + } rhs.map.each_map_entry([&](auto rhs_subspace, auto hash) { auto lhs_subspace = lhs.map.lookup(hash); @@ -412,9 +404,11 @@ FastValueIndex::sparse_only_merge(const ValueType &res_type, const Fun &fun, if (__builtin_expect((idx == result.my_cells.size), true)) { result.my_cells.push_back_fast(rhs_cells[rhs_subspace]); } + } else { + auto cell_value = fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace]); + *result.my_cells.get(lhs_subspace) = cell_value; } }); - return result; } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index ad182115054..f6c09f94fc9 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -253,7 +253,7 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { llvm::Value *eval_fun = builder.CreateIntToPtr(builder.getInt64((uint64_t)eval_ptr), eval_funptr_t, "inject_eval"); llvm::Value *ctx = builder.CreateIntToPtr(builder.getInt64((uint64_t)forest), builder.getVoidTy()->getPointerTo(), "inject_ctx"); if (pass_params == PassParams::ARRAY) { - push(builder.CreateCall(llvm::cast<llvm::FunctionType>(eval_fun->getType()->getPointerElementType()), + push(builder.CreateCall(llvm::cast<llvm::FunctionType>(eval_fun->getType()->getPointerElementType()), eval_fun, {ctx, params[0]}, "call_eval")); } else { assert(pass_params == PassParams::LAZY); diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 64b2b6f8865..98e3bc325cb 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -18,7 +18,6 @@ using Cells = SimpleTensor::Cells; using IndexList = std::vector<size_t>; using Label = SimpleTensor::Label; using CellRef = std::reference_wrapper<const Cell>; -using CellType = ValueType::CellType; namespace { diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index 766a4f1eb23..17faa635941 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -3,8 +3,6 @@ #include "simple_value.h" #include "inline_operation.h" #include <vespa/vespalib/util/typify.h> -#include <vespa/vespalib/util/visit_ranges.h> -#include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index 77ca6c1b8f0..614ef8389d8 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -125,7 +125,7 @@ void op_tensor_create(State &state, uint64_t param) { const Create &self = unwrap_param<Create>(param); TensorSpec spec(self.result_type().to_spec()); size_t i = 0; - for (auto pos = self.spec().rbegin(); pos != self.spec().rend(); ++pos) { + for (auto pos = self.map().rbegin(); pos != self.map().rend(); ++pos) { spec.add(pos->first, state.peek(i++).as_double()); } const Value &result = *state.stash.create<Value::UP>(state.engine.from_spec(spec)); @@ -180,7 +180,7 @@ void op_tensor_peek(State &state, uint64_t param) { const Peek &self = unwrap_param<Peek>(param); TensorSpec::Address addr; size_t child_cnt = 0; - for (auto pos = self.spec().rbegin(); pos != self.spec().rend(); ++pos) { + for (auto pos = self.map().rbegin(); pos != self.map().rend(); ++pos) { std::visit(vespalib::overload { [&](const TensorSpec::Label &label) { @@ -388,21 +388,27 @@ Concat::visit_self(vespalib::ObjectVisitor &visitor) const void Create::push_children(std::vector<Child::CREF> &children) const { - for (const auto &cell: _spec) { + for (const auto &cell: _map) { children.emplace_back(cell.second); } } +Create::Spec +Create::make_spec() const +{ + Spec generic_spec; + size_t child_idx = 0; + for (const auto & kv : map()) { + generic_spec[kv.first] = child_idx++; + } + return generic_spec; +} + Instruction Create::compile_self(EngineOrFactory engine, Stash &stash) const { if (engine.is_factory()) { - std::map<TensorSpec::Address, size_t> generic_spec; - size_t child_idx = 0; - for (const auto & kv : spec()) { - generic_spec[kv.first] = child_idx++; - } - return instruction::GenericCreate::make_instruction(result_type(), generic_spec, engine.factory(), stash); + return instruction::GenericCreate::make_instruction(result_type(), make_spec(), engine.factory(), stash); } return Instruction(op_tensor_create, wrap_param<Create>(*this)); } @@ -410,7 +416,7 @@ Create::compile_self(EngineOrFactory engine, Stash &stash) const void Create::visit_children(vespalib::ObjectVisitor &visitor) const { - for (const auto &cell: _spec) { + for (const auto &cell: _map) { ::visit(visitor, ::vespalib::eval::as_string(cell.first), cell.second.get()); } } @@ -487,7 +493,7 @@ void Peek::push_children(std::vector<Child::CREF> &children) const { children.emplace_back(_param); - for (const auto &dim: _spec) { + for (const auto &dim: _map) { std::visit(vespalib::overload { [&](const Child &child) { @@ -498,23 +504,29 @@ Peek::push_children(std::vector<Child::CREF> &children) const } } +Peek::Spec +Peek::make_spec() const +{ + Spec generic_spec; + size_t child_idx = 0; + for (const auto & [dim_name, label_or_child] : map()) { + std::visit(vespalib::overload { + [&,&dim_name = dim_name](const TensorSpec::Label &label) { + generic_spec.emplace(dim_name, label); + }, + [&,&dim_name = dim_name](const TensorFunction::Child &) { + generic_spec.emplace(dim_name, child_idx++); + } + }, label_or_child); + } + return generic_spec; +} + Instruction Peek::compile_self(EngineOrFactory engine, Stash &stash) const { if (engine.is_factory()) { - instruction::GenericPeek::SpecMap generic_spec; - size_t child_idx = 0; - for (const auto & [dim_name, label_or_child] : spec()) { - std::visit(vespalib::overload { - [&,&dim_name = dim_name](const TensorSpec::Label &label) { - generic_spec.emplace(dim_name, label); - }, - [&,&dim_name = dim_name](const TensorFunction::Child &) { - generic_spec.emplace(dim_name, child_idx++); - } - }, label_or_child); - } - return instruction::GenericPeek::make_instruction(param_type(), result_type(), generic_spec, engine.factory(), stash); + return instruction::GenericPeek::make_instruction(param_type(), result_type(), make_spec(), engine.factory(), stash); } return Instruction(op_tensor_peek, wrap_param<Peek>(*this)); } @@ -523,7 +535,7 @@ void Peek::visit_children(vespalib::ObjectVisitor &visitor) const { ::visit(visitor, "param", _param.get()); - for (const auto &dim: _spec) { + for (const auto &dim: _map) { std::visit(vespalib::overload { [&](const TensorSpec::Label &label) { diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index d6158f8eb4a..3c4eb6c53a4 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -310,16 +310,19 @@ class Create : public Node { using Super = Node; private: - std::map<TensorSpec::Address, Child> _spec; + std::map<TensorSpec::Address, Child> _map; public: Create(const ValueType &result_type_in, const std::map<TensorSpec::Address, TensorFunction::CREF> &spec_in) - : Super(result_type_in), _spec() + : Super(result_type_in), _map() { for (const auto &cell: spec_in) { - _spec.emplace(cell.first, Child(cell.second)); + _map.emplace(cell.first, Child(cell.second)); } } - const std::map<TensorSpec::Address, Child> &spec() const { return _spec; } + const std::map<TensorSpec::Address, Child> &map() const { return _map; } + // mapping from cell address to index of child that computes the cell value + using Spec = std::map<TensorSpec::Address, size_t>; + Spec make_spec() const; bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(EngineOrFactory engine, Stash &stash) const final override; void push_children(std::vector<Child::CREF> &children) const final override; @@ -359,25 +362,30 @@ public: using MyLabel = std::variant<TensorSpec::Label, Child>; private: Child _param; - std::map<vespalib::string, MyLabel> _spec; + std::map<vespalib::string, MyLabel> _map; public: Peek(const ValueType &result_type_in, const TensorFunction ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> &spec) - : Super(result_type_in), _param(param), _spec() + : Super(result_type_in), _param(param), _map() { for (const auto &dim: spec) { std::visit(vespalib::overload { [&](const TensorSpec::Label &label) { - _spec.emplace(dim.first, label); + _map.emplace(dim.first, label); }, [&](const TensorFunction::CREF &ref) { - _spec.emplace(dim.first, ref.get()); + _map.emplace(dim.first, ref.get()); } }, dim.second); } } - const std::map<vespalib::string, MyLabel> &spec() const { return _spec; } + const std::map<vespalib::string, MyLabel> &map() const { return _map; } + // a verbatim label or the index of a child that computes the label value: + using LabelOrChildIndex = std::variant<TensorSpec::Label, size_t>; + // mapping from dimension name to verbatim label or child index: + using Spec = std::map<vespalib::string, LabelOrChildIndex>; + Spec make_spec() const; const ValueType ¶m_type() const { return _param.get().result_type(); } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(EngineOrFactory engine, Stash &stash) const final override; diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt index 6e88beab9b7..f3b0750d503 100644 --- a/eval/src/vespa/eval/eval/test/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(eval_eval_test OBJECT SOURCES eval_fixture.cpp eval_spec.cpp + reference_operations.cpp tensor_conformance.cpp test_io.cpp value_compare.cpp diff --git a/eval/src/vespa/eval/eval/test/reference_operations.cpp b/eval/src/vespa/eval/eval/test/reference_operations.cpp new file mode 100644 index 00000000000..89b99526d55 --- /dev/null +++ b/eval/src/vespa/eval/eval/test/reference_operations.cpp @@ -0,0 +1,287 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "reference_operations.h" +#include <vespa/vespalib/util/overload.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> + +namespace vespalib::eval { + +namespace { + +bool concat_address(const TensorSpec::Address &me, const TensorSpec::Address &other, + const std::string &concat_dim, size_t my_offset, + TensorSpec::Address &my_out, TensorSpec::Address &other_out) +{ + my_out.insert_or_assign(concat_dim, my_offset); + for (const auto &my_dim: me) { + const auto & name = my_dim.first; + const auto & label = my_dim.second; + if (name == concat_dim) { + my_out.insert_or_assign(name, label.index + my_offset); + } else { + auto pos = other.find(name); + if ((pos == other.end()) || (pos->second == label)) { + my_out.insert_or_assign(name, label); + other_out.insert_or_assign(name, label); + } else { + return false; + } + } + } + return true; +} + +bool concat_addresses(const TensorSpec::Address &a, const TensorSpec::Address &b, + const std::string &concat_dim, size_t b_offset, + TensorSpec::Address &a_out, TensorSpec::Address &b_out) +{ + return concat_address(a, b, concat_dim, 0, a_out, b_out) && + concat_address(b, a, concat_dim, b_offset, b_out, a_out); +} + +double value_from_child(const TensorSpec &child) { + double sum = 0.0; + for (const auto & [addr, value] : child.cells()) { + sum += value; + } + return sum; +} + +bool join_address(const TensorSpec::Address &a, const TensorSpec::Address &b, TensorSpec::Address &addr) { + for (const auto &dim_a: a) { + auto pos_b = b.find(dim_a.first); + if ((pos_b != b.end()) && !(pos_b->second == dim_a.second)) { + return false; + } + addr.insert_or_assign(dim_a.first, dim_a.second); + } + return true; +} + +vespalib::string rename_dimension(const vespalib::string &name, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to) { + for (size_t i = 0; i < from.size(); ++i) { + if (name == from[i]) { + return to[i]; + } + } + return name; +} + +} // namespace <unnamed> + + +TensorSpec ReferenceOperations::concat(const TensorSpec &a, const TensorSpec &b, const std::string &concat_dim) { + ValueType a_type = ValueType::from_spec(a.type()); + ValueType b_type = ValueType::from_spec(b.type()); + ValueType res_type = ValueType::concat(a_type, b_type, concat_dim); + TensorSpec result(res_type.to_spec()); + if (res_type.is_error()) { + return result; + } + size_t b_offset = 1; + size_t concat_dim_index = a_type.dimension_index(concat_dim); + if (concat_dim_index != ValueType::Dimension::npos) { + const auto &dim = a_type.dimensions()[concat_dim_index]; + assert(dim.is_indexed()); // type resolving (above) should catch this + b_offset = dim.size; + } + for (const auto &cell_a: a.cells()) { + for (const auto &cell_b: b.cells()) { + TensorSpec::Address addr_a; + TensorSpec::Address addr_b; + if (concat_addresses(cell_a.first, cell_b.first, concat_dim, b_offset, addr_a, addr_b)) { + result.add(addr_a, cell_a.second); + result.add(addr_b, cell_b.second); + } + } + } + return result; +} + + +TensorSpec ReferenceOperations::create(const vespalib::string &type, const CreateSpec &spec, const std::vector<TensorSpec> &children) { + TensorSpec result(type); + if (ValueType::from_spec(type).is_error()) { + return result; + } + for (const auto & [addr, child_idx] : spec) { + assert(child_idx < children.size()); + const auto &child = children[child_idx]; + double val = value_from_child(child); + result.add(addr, val); + } + return result; +} + + +TensorSpec ReferenceOperations::join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + ValueType res_type = ValueType::join(ValueType::from_spec(a.type()), ValueType::from_spec(b.type())); + TensorSpec result(res_type.to_spec()); + if (res_type.is_error()) { + return result; + } + for (const auto &cell_a: a.cells()) { + for (const auto &cell_b: b.cells()) { + TensorSpec::Address addr; + if (join_address(cell_a.first, cell_b.first, addr) && + join_address(cell_b.first, cell_a.first, addr)) + { + result.add(addr, function(cell_a.second, cell_b.second)); + } + } + } + return result; +} + + +TensorSpec ReferenceOperations::map(const TensorSpec &a, map_fun_t func) { + ValueType res_type = ValueType::from_spec(a.type()); + TensorSpec result(res_type.to_spec()); + if (res_type.is_error()) { + return result; + } + for (const auto & [ addr, value ]: a.cells()) { + result.add(addr, func(value)); + } + return result; +} + + +TensorSpec ReferenceOperations::merge(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) { + ValueType res_type = ValueType::merge(ValueType::from_spec(a.type()), + ValueType::from_spec(b.type())); + TensorSpec result(res_type.to_spec()); + if (res_type.is_error()) { + return result; + } + for (const auto & [ addr, value ]: a.cells()) { + auto other = b.cells().find(addr); + if (other == b.cells().end()) { + result.add(addr, value); + } else { + result.add(addr, fun(value, other->second)); + } + } + for (const auto & [ addr, value ]: b.cells()) { + auto other = a.cells().find(addr); + if (other == a.cells().end()) { + result.add(addr, value); + } + } + return result; +} + + +TensorSpec ReferenceOperations::peek(const TensorSpec ¶m, const PeekSpec &peek_spec, const std::vector<TensorSpec> &children) { + if (peek_spec.empty()) { + return TensorSpec(ValueType::error_type().to_spec()); + } + std::vector<vespalib::string> peek_dims; + for (const auto & [dim_name, label_or_child] : peek_spec) { + peek_dims.push_back(dim_name); + } + ValueType param_type = ValueType::from_spec(param.type()); + ValueType result_type = param_type.reduce(peek_dims); + TensorSpec result(result_type.to_spec()); + if (result_type.is_error()) { + return result; + } + auto is_mapped_dim = [&](const vespalib::string &name) { + size_t dim_idx = param_type.dimension_index(name); + assert(dim_idx != ValueType::Dimension::npos); + const auto ¶m_dim = param_type.dimensions()[dim_idx]; + return param_dim.is_mapped(); + }; + TensorSpec::Address addr; + for (const auto & [dim_name, label_or_child] : peek_spec) { + const vespalib::string &dim = dim_name; + std::visit(vespalib::overload + { + [&](const TensorSpec::Label &label) { + addr.emplace(dim, label); + }, + [&](const size_t &child_idx) { + assert(child_idx < children.size()); + const auto &child = children[child_idx]; + double child_value = value_from_child(child); + if (is_mapped_dim(dim)) { + addr.emplace(dim, vespalib::make_string("%zd", int64_t(child_value))); + } else { + addr.emplace(dim, child_value); + } + } + }, label_or_child); + } + for (const auto &cell: param.cells()) { + bool keep = true; + TensorSpec::Address my_addr; + for (const auto &binding: cell.first) { + auto pos = addr.find(binding.first); + if (pos == addr.end()) { + my_addr.emplace(binding.first, binding.second); + } else { + if (!(pos->second == binding.second)) { + keep = false; + } + } + } + if (keep) { + result.add(my_addr, cell.second); + } + } + return result; +} + + +TensorSpec ReferenceOperations::reduce(const TensorSpec &a, const std::vector<vespalib::string> &dims, Aggr aggr) { + ValueType res_type = ValueType::from_spec(a.type()).reduce(dims); + TensorSpec result(res_type.to_spec()); + if (res_type.is_error()) { + return result; + } + Stash stash; + std::map<TensorSpec::Address,std::optional<Aggregator*>> my_map; + for (const auto &cell: a.cells()) { + TensorSpec::Address addr; + for (const auto &dim: cell.first) { + if (res_type.dimension_index(dim.first) != ValueType::Dimension::npos) { + addr.insert_or_assign(dim.first, dim.second); + } + } + auto [pos, is_empty] = my_map.emplace(addr, std::nullopt); + if (is_empty) { + pos->second = &Aggregator::create(aggr, stash); + pos->second.value()->first(cell.second); + } else { + pos->second.value()->next(cell.second); + } + } + for (const auto &my_entry: my_map) { + result.add(my_entry.first, my_entry.second.value()->result()); + } + return result; +} + + +TensorSpec ReferenceOperations::rename(const TensorSpec &a, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to) { + assert(from.size() == to.size()); + ValueType res_type = ValueType::from_spec(a.type()).rename(from, to); + TensorSpec result(res_type.to_spec()); + if (res_type.is_error()) { + return result; + } + for (const auto &cell: a.cells()) { + TensorSpec::Address addr; + for (const auto &dim: cell.first) { + addr.insert_or_assign(rename_dimension(dim.first, from, to), dim.second); + } + result.add(addr, cell.second); + } + return result; +} + + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/reference_operations.h b/eval/src/vespa/eval/eval/test/reference_operations.h new file mode 100644 index 00000000000..735454b486a --- /dev/null +++ b/eval/src/vespa/eval/eval/test/reference_operations.h @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/aggr.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/tensor_function.h> + +#include <vector> +#include <map> +#include <variant> + +namespace vespalib::eval { + +struct ReferenceOperations { + using map_fun_t = vespalib::eval::operation::op1_t; + using join_fun_t = vespalib::eval::operation::op2_t; + + // mapping from cell address to index of child that computes the cell value + using CreateSpec = tensor_function::Create::Spec; + + // mapping from dimension name to verbatim label or child + using PeekSpec = tensor_function::Peek::Spec; + + static TensorSpec concat(const TensorSpec &a, const TensorSpec &b, const std::string &concat_dim); + static TensorSpec create(const vespalib::string &type, const CreateSpec &spec, const std::vector<TensorSpec> &children); + static TensorSpec join(const TensorSpec &a, const TensorSpec &b, join_fun_t function); + static TensorSpec map(const TensorSpec &a, map_fun_t func); + static TensorSpec merge(const TensorSpec &a, const TensorSpec &b, join_fun_t fun); + static TensorSpec peek(const TensorSpec ¶m, const PeekSpec &spec, const std::vector<TensorSpec> &children); + static TensorSpec reduce(const TensorSpec &a, const std::vector<vespalib::string> &dims, Aggr aggr); + static TensorSpec rename(const TensorSpec &a, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to); +}; + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/tensor_model.hpp b/eval/src/vespa/eval/eval/test/tensor_model.hpp index 59653954c9e..78d6798ac4c 100644 --- a/eval/src/vespa/eval/eval/test/tensor_model.hpp +++ b/eval/src/vespa/eval/eval/test/tensor_model.hpp @@ -16,7 +16,6 @@ namespace vespalib { namespace eval { namespace test { -using CellType = ValueType::CellType; using map_fun_t = vespalib::eval::operation::op1_t; using join_fun_t = vespalib::eval::operation::op2_t; diff --git a/eval/src/vespa/eval/eval/typed_cells.h b/eval/src/vespa/eval/eval/typed_cells.h index 09d5c080cf7..a478a419f95 100644 --- a/eval/src/vespa/eval/eval/typed_cells.h +++ b/eval/src/vespa/eval/eval/typed_cells.h @@ -11,8 +11,6 @@ namespace vespalib::eval { // Low-level typed cells reference struct TypedCells { - using CellType = vespalib::eval::ValueType::CellType; - const void *data; CellType type; size_t size:56; diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index 2de95657f72..923d3f29cd3 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -14,8 +14,6 @@ namespace vespalib::eval { namespace { -using CellType = ValueType::CellType; - constexpr uint32_t DOUBLE_CELL_TYPE = 0; constexpr uint32_t FLOAT_CELL_TYPE = 1; @@ -118,7 +116,7 @@ ValueType decode_type(nbostream &input, const Format &format) { } } if (dim_list.empty()) { - assert(cell_type == ValueType::CellType::DOUBLE); + assert(cell_type == CellType::DOUBLE); } return ValueType::tensor_type(std::move(dim_list), cell_type); } diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index c7d77c766bc..05ec65bf292 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -8,7 +8,6 @@ namespace vespalib::eval { namespace { -using CellType = ValueType::CellType; using Dimension = ValueType::Dimension; using DimensionList = std::vector<Dimension>; diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index ae69b5a3349..6d9316e76ed 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -16,7 +16,6 @@ namespace vespalib::eval { class ValueType { public: - using CellType = vespalib::eval::CellType; struct Dimension { using size_type = uint32_t; static constexpr size_type npos = -1; diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp index 847203db3b1..a4575e33c2f 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.cpp +++ b/eval/src/vespa/eval/eval/value_type_spec.cpp @@ -8,8 +8,6 @@ namespace vespalib::eval::value_type { -using CellType = ValueType::CellType; - namespace { const char *to_name(CellType cell_type) { @@ -188,7 +186,7 @@ parse_spec(const char *pos_in, const char *end_in, const char *&pos_out, } else if (type_name == "float") { return ValueType::make_type(CellType::FLOAT, {}); } else if (type_name == "tensor") { - ValueType::CellType cell_type = parse_cell_type(ctx); + CellType cell_type = parse_cell_type(ctx); std::vector<ValueType::Dimension> list = parse_dimension_list(ctx); if (!ctx.failed()) { if (unsorted != nullptr) { diff --git a/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp b/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp index cc746c4db83..5dcfcba025d 100644 --- a/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp @@ -45,12 +45,12 @@ struct MyDotProductOp { static auto invoke() { return my_dot_product_op<LCT,RCT>; } }; -InterpretedFunction::op_function my_select(ValueType::CellType lct, ValueType::CellType rct) { +InterpretedFunction::op_function my_select(CellType lct, CellType rct) { if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { + if (lct == CellType::DOUBLE) { return my_cblas_double_dot_product_op; } - if (lct == ValueType::CellType::FLOAT) { + if (lct == CellType::FLOAT) { return my_cblas_float_dot_product_op; } } diff --git a/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp b/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp index 0abcd452645..5e9ff6a0ef0 100644 --- a/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_lambda_peek_function.cpp @@ -27,7 +27,7 @@ void my_lambda_peek_op(InterpretedFunction::State &state, uint64_t param) { const auto &self = unwrap_param<Self>(param); const std::vector<uint32_t> &lookup_table = self.table_token->get(); auto src_cells = state.peek(0).cells().typify<SRC_CT>(); - ArrayRef<DST_CT> dst_cells = state.stash.create_array<DST_CT>(lookup_table.size()); + ArrayRef<DST_CT> dst_cells = state.stash.create_uninitialized_array<DST_CT>(lookup_table.size()); DST_CT *dst = &dst_cells[0]; for (uint32_t idx: lookup_table) { *dst++ = src_cells[idx]; diff --git a/eval/src/vespa/eval/instruction/dense_matmul_function.cpp b/eval/src/vespa/eval/instruction/dense_matmul_function.cpp index 554122a67b4..5d4ebb88931 100644 --- a/eval/src/vespa/eval/instruction/dense_matmul_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_matmul_function.cpp @@ -32,7 +32,7 @@ void my_matmul_op(InterpretedFunction::State &state, uint64_t param) { using OCT = typename UnifyCellTypes<LCT,RCT>::type; auto lhs_cells = state.peek(1).cells().typify<LCT>(); auto rhs_cells = state.peek(0).cells().typify<RCT>(); - auto dst_cells = state.stash.create_array<OCT>(self.lhs_size * self.rhs_size); + auto dst_cells = state.stash.create_uninitialized_array<OCT>(self.lhs_size * self.rhs_size); OCT *dst = dst_cells.begin(); const LCT *lhs = lhs_cells.cbegin(); for (size_t i = 0; i < self.lhs_size; ++i) { diff --git a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp index cbca2ff14f2..42e7deb9523 100644 --- a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp @@ -60,11 +60,11 @@ void my_cblas_float_multi_matmul_op(InterpretedFunction::State &state, uint64_t state.pop_pop_push(state.stash.create<tensor::DenseTensorView>(self.result_type(), TypedCells(dst_cells))); } -InterpretedFunction::op_function my_select(ValueType::CellType cell_type) { - if (cell_type == ValueType::CellType::DOUBLE) { +InterpretedFunction::op_function my_select(CellType cell_type) { + if (cell_type == CellType::DOUBLE) { return my_cblas_double_multi_matmul_op; } - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return my_cblas_float_multi_matmul_op; } abort(); @@ -117,7 +117,7 @@ struct DimPrefix { bool check_input_type(const ValueType &type, const DimList &relevant) { return (type.is_dense() && (relevant.size() >= 2) && - ((type.cell_type() == ValueType::CellType::FLOAT) || (type.cell_type() == ValueType::CellType::DOUBLE))); + ((type.cell_type() == CellType::FLOAT) || (type.cell_type() == CellType::DOUBLE))); } bool is_multi_matmul(const ValueType &a, const ValueType &b, const vespalib::string &reduce_dim) { diff --git a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp index daad4da947b..fd93cd62fa9 100644 --- a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.cpp @@ -75,10 +75,10 @@ DenseTensorPeekFunction::optimize(const TensorFunction &expr, Stash &stash) const ValueType &peek_type = peek->param_type(); if (expr.result_type().is_double() && peek_type.is_dense()) { std::vector<std::pair<int64_t,size_t>> spec; - assert(peek_type.dimensions().size() == peek->spec().size()); + assert(peek_type.dimensions().size() == peek->map().size()); for (auto dim = peek_type.dimensions().rbegin(); dim != peek_type.dimensions().rend(); ++dim) { - auto dim_spec = peek->spec().find(dim->name); - assert(dim_spec != peek->spec().end()); + auto dim_spec = peek->map().find(dim->name); + assert(dim_spec != peek->map().end()); std::visit(vespalib::overload { diff --git a/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp b/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp index fdcef97e277..44332bbacee 100644 --- a/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_xw_product_function.cpp @@ -33,7 +33,7 @@ void my_xw_product_op(InterpretedFunction::State &state, uint64_t param) { using OCT = typename UnifyCellTypes<LCT,RCT>::type; auto vector_cells = state.peek(1).cells().typify<LCT>(); auto matrix_cells = state.peek(0).cells().typify<RCT>(); - auto dst_cells = state.stash.create_array<OCT>(self.result_size); + auto dst_cells = state.stash.create_uninitialized_array<OCT>(self.result_size); OCT *dst = dst_cells.begin(); const RCT *matrix = matrix_cells.cbegin(); for (size_t i = 0; i < self.result_size; ++i) { diff --git a/eval/src/vespa/eval/instruction/generic_create.h b/eval/src/vespa/eval/instruction/generic_create.h index dc3cebc1086..dfd858613fe 100644 --- a/eval/src/vespa/eval/instruction/generic_create.h +++ b/eval/src/vespa/eval/instruction/generic_create.h @@ -5,6 +5,7 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/tensor_function.h> #include <map> namespace vespalib { class Stash; } @@ -15,7 +16,8 @@ namespace vespalib::eval::instruction { //----------------------------------------------------------------------------- struct GenericCreate { - using SpecMap = std::map<TensorSpec::Address, size_t>; + // mapping from cell address to index of child that computes the cell value + using SpecMap = tensor_function::Create::Spec; static InterpretedFunction::Instruction make_instruction(const ValueType &res_type, diff --git a/eval/src/vespa/eval/instruction/generic_lambda.cpp b/eval/src/vespa/eval/instruction/generic_lambda.cpp index 1ba2f710909..5685f199b9e 100644 --- a/eval/src/vespa/eval/instruction/generic_lambda.cpp +++ b/eval/src/vespa/eval/instruction/generic_lambda.cpp @@ -103,7 +103,7 @@ void my_interpreted_lambda_op(eval::InterpretedFunction::State &state, uint64_t std::vector<double> labels(params.result_type.dimensions().size(), 0.0); ParamProxy param_proxy(labels, *state.params, params.bindings); InterpretedFunction::Context ctx(params.fun); - ArrayRef<CT> dst_cells = state.stash.create_array<CT>(params.num_cells); + ArrayRef<CT> dst_cells = state.stash.create_uninitialized_array<CT>(params.num_cells); CT *dst = &dst_cells[0]; do { *dst++ = params.fun.eval(ctx, param_proxy).as_double(); diff --git a/eval/src/vespa/eval/instruction/generic_merge.cpp b/eval/src/vespa/eval/instruction/generic_merge.cpp index 87be47a9c2e..8de4ea1adeb 100644 --- a/eval/src/vespa/eval/instruction/generic_merge.cpp +++ b/eval/src/vespa/eval/instruction/generic_merge.cpp @@ -127,9 +127,19 @@ void my_sparse_merge_op(State &state, uint64_t param_in) { if (auto indexes = detect_type<FastValueIndex>(lhs.index(), rhs.index())) { auto lhs_cells = lhs.cells().typify<LCT>(); auto rhs_cells = rhs.cells().typify<RCT>(); - return state.pop_pop_push( + if (lhs_cells.size() < rhs_cells.size()) { + return state.pop_pop_push( + FastValueIndex::sparse_only_merge<RCT,LCT,OCT,Fun>( + param.res_type, Fun(param.function), + indexes.get<1>(), indexes.get<0>(), + rhs_cells, lhs_cells, state.stash)); + } else { + return state.pop_pop_push( FastValueIndex::sparse_only_merge<LCT,RCT,OCT,Fun>( - param.res_type, Fun(param.function), indexes.get<0>(), indexes.get<1>(), lhs_cells, rhs_cells, state.stash)); + param.res_type, Fun(param.function), + indexes.get<0>(), indexes.get<1>(), + lhs_cells, rhs_cells, state.stash)); + } } auto up = generic_mixed_merge<LCT, RCT, OCT, Fun>(lhs, rhs, param); auto &result = state.stash.create<std::unique_ptr<Value>>(std::move(up)); diff --git a/eval/src/vespa/eval/instruction/generic_peek.cpp b/eval/src/vespa/eval/instruction/generic_peek.cpp index 5802a60d43a..d8ae9241f44 100644 --- a/eval/src/vespa/eval/instruction/generic_peek.cpp +++ b/eval/src/vespa/eval/instruction/generic_peek.cpp @@ -35,7 +35,7 @@ size_t count_children(const Spec &spec) struct DimSpec { vespalib::stringref name; - GenericPeek::MyLabel child_or_label; + GenericPeek::SpecMap::mapped_type child_or_label; bool has_child() const { return std::holds_alternative<size_t>(child_or_label); } diff --git a/eval/src/vespa/eval/instruction/generic_peek.h b/eval/src/vespa/eval/instruction/generic_peek.h index d31b47238cb..3fe7aa9d270 100644 --- a/eval/src/vespa/eval/instruction/generic_peek.h +++ b/eval/src/vespa/eval/instruction/generic_peek.h @@ -5,6 +5,7 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/tensor_function.h> #include <map> namespace vespalib { class Stash; } @@ -15,8 +16,8 @@ namespace vespalib::eval::instruction { //----------------------------------------------------------------------------- struct GenericPeek { - using MyLabel = std::variant<TensorSpec::Label, size_t>; - using SpecMap = std::map<vespalib::string, MyLabel>; + // mapping from dimension name to verbatim label or child + using SpecMap = tensor_function::Peek::Spec; static InterpretedFunction::Instruction make_instruction(const ValueType &input_type, diff --git a/eval/src/vespa/eval/streamed/CMakeLists.txt b/eval/src/vespa/eval/streamed/CMakeLists.txt new file mode 100644 index 00000000000..ee928d7b2c9 --- /dev/null +++ b/eval/src/vespa/eval/streamed/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_library(eval_streamed OBJECT + SOURCES + streamed_value.cpp + streamed_value_index.cpp + streamed_value_utils.cpp + streamed_value_builder.cpp + streamed_value_builder_factory.cpp + streamed_value_view.cpp +) diff --git a/eval/src/vespa/eval/streamed/streamed_value.cpp b/eval/src/vespa/eval/streamed/streamed_value.cpp new file mode 100644 index 00000000000..bdfe5fd4e27 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value.cpp @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value.h" +#include <vespa/log/log.h> + +LOG_SETUP(".vespalib.eval.streamed.streamed_value"); + +namespace vespalib::eval { + +template <typename T> +StreamedValue<T>::~StreamedValue() = default; + +template <typename T> +MemoryUsage +StreamedValue<T>::get_memory_usage() const +{ + MemoryUsage usage = self_memory_usage<StreamedValue<T>>(); + usage.merge(vector_extra_memory_usage(_my_cells)); + usage.incUsedBytes(_label_buf.byteSize()); + usage.incAllocatedBytes(_label_buf.byteCapacity()); + return usage; +} + +template class StreamedValue<double>; +template class StreamedValue<float>; + +} // namespace + diff --git a/eval/src/vespa/eval/streamed/streamed_value.h b/eval/src/vespa/eval/streamed/streamed_value.h new file mode 100644 index 00000000000..258802a53e8 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value.h @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/value.h> +#include "streamed_value_index.h" +#include <cassert> + +namespace vespalib::eval { + +/** + * A very simple Value implementation. + * Cheap to construct from serialized data, + * and cheap to serialize or iterate through. + * Slow for full or partial lookups. + **/ +template <typename T> +class StreamedValue : public Value +{ +private: + ValueType _type; + std::vector<T> _my_cells; + Array<char> _label_buf; + StreamedValueIndex _my_index; + +public: + StreamedValue(ValueType type, size_t num_mapped_dimensions, + std::vector<T> cells, size_t num_subspaces, Array<char> && label_buf) + : _type(std::move(type)), + _my_cells(std::move(cells)), + _label_buf(std::move(label_buf)), + _my_index(num_mapped_dimensions, + num_subspaces, + ConstArrayRef<char>(_label_buf.begin(), _label_buf.size())) + { + assert(num_subspaces * _type.dense_subspace_size() == _my_cells.size()); + } + + ~StreamedValue(); + const ValueType &type() const final override { return _type; } + TypedCells cells() const final override { return TypedCells(_my_cells); } + const Value::Index &index() const final override { return _my_index; } + MemoryUsage get_memory_usage() const final override; + auto get_data_reference() const { return _my_index.get_data_reference(); } +}; + +} // namespace diff --git a/eval/src/vespa/eval/streamed/streamed_value_builder.cpp b/eval/src/vespa/eval/streamed/streamed_value_builder.cpp new file mode 100644 index 00000000000..957121c42b7 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_builder.cpp @@ -0,0 +1,13 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value_builder.h" + +namespace vespalib::eval { + +template<typename T> +StreamedValueBuilder<T>::~StreamedValueBuilder() = default; + +template class StreamedValueBuilder<double>; +template class StreamedValueBuilder<float>; + +} // namespace diff --git a/eval/src/vespa/eval/streamed/streamed_value_builder.h b/eval/src/vespa/eval/streamed/streamed_value_builder.h new file mode 100644 index 00000000000..5698c805756 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_builder.h @@ -0,0 +1,66 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "streamed_value.h" +#include <vespa/vespalib/objects/nbostream.h> + +namespace vespalib::eval { + + /** + * Builder for StreamedValue objects. + **/ +template <typename T> +class StreamedValueBuilder : public ValueBuilder<T> +{ +private: + ValueType _type; + size_t _num_mapped_dimensions; + size_t _dense_subspace_size; + std::vector<T> _cells; + size_t _num_subspaces; + nbostream _labels; +public: + StreamedValueBuilder(const ValueType &type, + size_t num_mapped_in, + size_t subspace_size_in, + size_t expected_subspaces) + : _type(type), + _num_mapped_dimensions(num_mapped_in), + _dense_subspace_size(subspace_size_in), + _cells(), + _num_subspaces(0), + _labels() + { + _cells.reserve(subspace_size_in * expected_subspaces); + // assume small sized label strings: + _labels.reserve(num_mapped_in * expected_subspaces * 3); + }; + + ~StreamedValueBuilder(); + + ArrayRef<T> add_subspace(ConstArrayRef<vespalib::stringref> addr) override { + for (auto label : addr) { + _labels.writeSmallString(label); + } + size_t old_sz = _cells.size(); + _cells.resize(old_sz + _dense_subspace_size); + _num_subspaces++; + return ArrayRef<T>(&_cells[old_sz], _dense_subspace_size); + } + + std::unique_ptr<Value> build(std::unique_ptr<ValueBuilder<T>>) override { + if (_num_mapped_dimensions == 0) { + assert(_num_subspaces == 1); + } + assert(_num_subspaces * _dense_subspace_size == _cells.size()); + return std::make_unique<StreamedValue<T>>(std::move(_type), + _num_mapped_dimensions, + std::move(_cells), + _num_subspaces, + _labels.extract_buffer()); + } + +}; + +} // namespace diff --git a/eval/src/vespa/eval/streamed/streamed_value_builder_factory.cpp b/eval/src/vespa/eval/streamed/streamed_value_builder_factory.cpp new file mode 100644 index 00000000000..aa6347a2c51 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_builder_factory.cpp @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value_builder_factory.h" +#include "streamed_value_builder.h" + +namespace vespalib::eval { + +struct SelectStreamedValueBuilder { + template <typename T> + static std::unique_ptr<ValueBuilderBase> invoke( + const ValueType &type, size_t num_mapped, + size_t subspace_size, size_t expected_subspaces) + { + assert(check_cell_type<T>(type.cell_type())); + return std::make_unique<StreamedValueBuilder<T>>( + type, num_mapped, subspace_size, expected_subspaces); + } +}; + +std::unique_ptr<ValueBuilderBase> +StreamedValueBuilderFactory::create_value_builder_base(const ValueType &type, + size_t num_mapped, + size_t subspace_size, + size_t expected_subspaces) const +{ + return typify_invoke<1,TypifyCellType,SelectStreamedValueBuilder>( + type.cell_type(), + type, num_mapped, subspace_size, expected_subspaces); +} + +StreamedValueBuilderFactory::~StreamedValueBuilderFactory() = default; +StreamedValueBuilderFactory StreamedValueBuilderFactory::_factory; + +} // namespace + + diff --git a/eval/src/vespa/eval/streamed/streamed_value_builder_factory.h b/eval/src/vespa/eval/streamed/streamed_value_builder_factory.h new file mode 100644 index 00000000000..3f81981f429 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_builder_factory.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "streamed_value.h" + +namespace vespalib::eval { + +/** + * A factory that can generate appropriate ValueBuilder instances + */ +struct StreamedValueBuilderFactory : ValueBuilderFactory { +private: + StreamedValueBuilderFactory() {} + static StreamedValueBuilderFactory _factory; + std::unique_ptr<ValueBuilderBase> create_value_builder_base( + const ValueType &type, size_t num_mapped_in, + size_t subspace_size_in, size_t expected_subspaces) const override; +public: + static const StreamedValueBuilderFactory &get() { return _factory; } + ~StreamedValueBuilderFactory(); +}; + +} diff --git a/eval/src/vespa/eval/streamed/streamed_value_index.cpp b/eval/src/vespa/eval/streamed/streamed_value_index.cpp new file mode 100644 index 00000000000..38b57e9c660 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_index.cpp @@ -0,0 +1,100 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value_index.h" +#include "streamed_value_utils.h" + +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.tensor.streamed_value_index"); + +namespace vespalib::eval { + +namespace { + +struct StreamedFilterView : Value::Index::View +{ + LabelBlockStream label_blocks; + std::vector<size_t> view_dims; + std::vector<vespalib::stringref> to_match; + + StreamedFilterView(LabelBlockStream labels, std::vector<size_t> view_dims_in) + : label_blocks(std::move(labels)), + view_dims(std::move(view_dims_in)), + to_match() + { + to_match.reserve(view_dims.size()); + } + + void lookup(ConstArrayRef<const vespalib::stringref*> addr) override { + label_blocks.reset(); + to_match.clear(); + for (auto ptr : addr) { + to_match.push_back(*ptr); + } + assert(view_dims.size() == to_match.size()); + } + + bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override { + while (const auto block = label_blocks.next_block()) { + idx_out = block.ss_idx; + bool matches = true; + size_t out_idx = 0; + size_t vdm_idx = 0; + for (size_t dim = 0; dim < block.address.size(); ++dim) { + if (vdm_idx < view_dims.size() && (view_dims[vdm_idx] == dim)) { + matches &= (block.address[dim] == to_match[vdm_idx++]); + } else { + *addr_out[out_idx++] = block.address[dim]; + } + } + assert(out_idx == addr_out.size()); + assert(vdm_idx == view_dims.size()); + if (matches) return true; + } + return false; + } +}; + +struct StreamedIterationView : Value::Index::View +{ + LabelBlockStream label_blocks; + + StreamedIterationView(LabelBlockStream labels) + : label_blocks(std::move(labels)) + {} + + void lookup(ConstArrayRef<const vespalib::stringref*> addr) override { + label_blocks.reset(); + assert(addr.size() == 0); + } + + bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override { + if (auto block = label_blocks.next_block()) { + idx_out = block.ss_idx; + size_t i = 0; + assert(addr_out.size() == block.address.size()); + for (auto ptr : addr_out) { + *ptr = block.address[i++]; + } + return true; + } + return false; + } +}; + +} // namespace <unnamed> + +std::unique_ptr<Value::Index::View> +StreamedValueIndex::create_view(const std::vector<size_t> &dims) const +{ + LabelBlockStream label_stream(_data.num_subspaces, _data.labels_buffer, _data.num_mapped_dims); + if (dims.empty()) { + return std::make_unique<StreamedIterationView>(std::move(label_stream)); + } + return std::make_unique<StreamedFilterView>(std::move(label_stream), dims); +} + +} // namespace vespalib::eval diff --git a/eval/src/vespa/eval/streamed/streamed_value_index.h b/eval/src/vespa/eval/streamed/streamed_value_index.h new file mode 100644 index 00000000000..8fd561200c3 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_index.h @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value.h> + +namespace vespalib::eval { + + /** + * Implements Value::Index by reading a stream of serialized + * labels. + **/ +class StreamedValueIndex : public Value::Index +{ +public: + struct SerializedDataRef { + uint32_t num_mapped_dims; + uint32_t num_subspaces; + ConstArrayRef<char> labels_buffer; + }; + StreamedValueIndex(uint32_t num_mapped_dims, uint32_t num_subspaces, ConstArrayRef<char> labels_buf) + : _data{num_mapped_dims, num_subspaces, labels_buf} + {} + + // index API: + size_t size() const override { return _data.num_subspaces; } + std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; + + SerializedDataRef get_data_reference() const { return _data; } + +private: + SerializedDataRef _data; +}; + +} // namespace + diff --git a/eval/src/vespa/eval/streamed/streamed_value_utils.cpp b/eval/src/vespa/eval/streamed/streamed_value_utils.cpp new file mode 100644 index 00000000000..1b4a91a9080 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_utils.cpp @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value_utils.h" + +namespace vespalib::eval { + +LabelBlockStream::~LabelBlockStream() = default; + +} // namespace diff --git a/eval/src/vespa/eval/streamed/streamed_value_utils.h b/eval/src/vespa/eval/streamed/streamed_value_utils.h new file mode 100644 index 00000000000..3e3da82dd22 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_utils.h @@ -0,0 +1,76 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value.h> +#include <vespa/vespalib/objects/nbostream.h> + +namespace vespalib::eval { + +/** + * Reads a stream of serialized labels. + * Reading more labels than available will + * throw an exception. + **/ +struct LabelStream { + nbostream source; + LabelStream(ConstArrayRef<char> data) : source(data.begin(), data.size()) {} + vespalib::stringref next_label() { + size_t str_size = source.getInt1_4Bytes(); + vespalib::stringref label(source.peek(), str_size); + source.adjustReadPos(str_size); + return label; + } + void reset() { source.rp(0); } +}; + +/** + * Represents an address (set of labels) mapping to a subspace index + **/ +struct LabelBlock { + static constexpr size_t npos = -1; + size_t ss_idx; + ConstArrayRef<vespalib::stringref> address; + operator bool() const { return ss_idx != npos; } +}; + +/** + * Utility for reading a buffer with serialized labels + * as a stream of LabelBlock objects. + **/ +class LabelBlockStream { +private: + size_t _num_subspaces; + LabelStream _labels; + size_t _subspace_index; + std::vector<vespalib::stringref> _current_address; +public: + LabelBlock next_block() { + if (_subspace_index < _num_subspaces) { + for (auto & label : _current_address) { + label = _labels.next_label(); + } + return LabelBlock{_subspace_index++, _current_address}; + } else { + return LabelBlock{LabelBlock::npos, {}}; + } + } + + void reset() { + _subspace_index = 0; + _labels.reset(); + } + + LabelBlockStream(uint32_t num_subspaces, + ConstArrayRef<char> label_buf, + uint32_t num_mapped_dims) + : _num_subspaces(num_subspaces), + _labels(label_buf), + _subspace_index(num_subspaces), + _current_address(num_mapped_dims) + {} + + ~LabelBlockStream(); +}; + +} // namespace diff --git a/eval/src/vespa/eval/streamed/streamed_value_view.cpp b/eval/src/vespa/eval/streamed/streamed_value_view.cpp new file mode 100644 index 00000000000..87e1e676692 --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_view.cpp @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "streamed_value_view.h" + +namespace vespalib::eval { + +StreamedValueView::~StreamedValueView() = default; + +} // namespace diff --git a/eval/src/vespa/eval/streamed/streamed_value_view.h b/eval/src/vespa/eval/streamed/streamed_value_view.h new file mode 100644 index 00000000000..e37f442dd9a --- /dev/null +++ b/eval/src/vespa/eval/streamed/streamed_value_view.h @@ -0,0 +1,45 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/value.h> +#include "streamed_value_index.h" +#include <cassert> + +namespace vespalib::eval { + + /** + * Same characteristics as StreamedValue, but does not + * own its data - refers to type, cells and serialized + * labels that must be kept outside the Value. + **/ +class StreamedValueView : public Value +{ +private: + const ValueType &_type; + TypedCells _cells_ref; + StreamedValueIndex _my_index; + +public: + StreamedValueView(const ValueType &type, size_t num_mapped_dimensions, + TypedCells cells, size_t num_subspaces, + ConstArrayRef<char> labels_buf) + : _type(type), + _cells_ref(cells), + _my_index(num_mapped_dimensions, num_subspaces, labels_buf) + { + assert(num_subspaces * _type.dense_subspace_size() == _cells_ref.size); + } + + ~StreamedValueView(); + const ValueType &type() const final override { return _type; } + TypedCells cells() const final override { return _cells_ref; } + const Value::Index &index() const final override { return _my_index; } + MemoryUsage get_memory_usage() const final override { + return self_memory_usage<StreamedValueView>(); + } + auto get_data_reference() const { return _my_index.get_data_reference(); } +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/CMakeLists.txt b/eval/src/vespa/eval/tensor/CMakeLists.txt index 77ae1daec88..75be3e802ea 100644 --- a/eval/src/vespa/eval/tensor/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_add_library(eval_tensor OBJECT SOURCES default_tensor_engine.cpp - default_value_builder_factory.cpp partial_update.cpp tensor.cpp tensor_address.cpp diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index 04aad776e43..68c8dc990c6 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -46,7 +46,6 @@ using eval::TensorFunction; using eval::TensorSpec; using eval::Value; using eval::ValueType; -using CellType = eval::ValueType::CellType; using vespalib::IllegalArgumentException; using vespalib::make_string; @@ -451,7 +450,7 @@ void append_vector(OCT *&pos, const Value &value) { template <typename OCT> const Value &concat_vectors(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { - ArrayRef<OCT> cells = stash.create_array<OCT>(vector_size); + ArrayRef<OCT> cells = stash.create_uninitialized_array<OCT>(vector_size); OCT *pos = cells.begin(); append_vector<OCT>(pos, a); append_vector<OCT>(pos, b); diff --git a/eval/src/vespa/eval/tensor/default_value_builder_factory.cpp b/eval/src/vespa/eval/tensor/default_value_builder_factory.cpp deleted file mode 100644 index 74fd371e9a0..00000000000 --- a/eval/src/vespa/eval/tensor/default_value_builder_factory.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "default_value_builder_factory.h" -#include <vespa/vespalib/util/typify.h> -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/double_value_builder.h> -#include <vespa/eval/tensor/dense/dense_tensor_value_builder.h> -#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h> -#include <vespa/eval/tensor/sparse/sparse_tensor_value_builder.h> - -using namespace vespalib::eval; - -namespace vespalib::tensor { - -//----------------------------------------------------------------------------- - -namespace { - -struct CreateDefaultValueBuilderBase { - template <typename T> static std::unique_ptr<ValueBuilderBase> invoke(const ValueType &type, - size_t num_mapped_dims, - size_t subspace_size, - size_t expected_subspaces) - { - assert(check_cell_type<T>(type.cell_type())); - if (type.is_double()) { - return std::make_unique<DoubleValueBuilder>(); - } - if (num_mapped_dims == 0) { - return std::make_unique<DenseTensorValueBuilder<T>>(type, subspace_size); - } - if (subspace_size == 1) { - return std::make_unique<SparseTensorValueBuilder<T>>(type, num_mapped_dims, expected_subspaces); - } - return std::make_unique<packed_mixed_tensor::PackedMixedTensorBuilder<T>>(type, num_mapped_dims, subspace_size, expected_subspaces); - } -}; - -} // namespace <unnamed> - -//----------------------------------------------------------------------------- - -DefaultValueBuilderFactory::DefaultValueBuilderFactory() = default; -DefaultValueBuilderFactory DefaultValueBuilderFactory::_factory; - -std::unique_ptr<ValueBuilderBase> -DefaultValueBuilderFactory::create_value_builder_base(const ValueType &type, - size_t num_mapped_dims, - size_t subspace_size, - size_t expected_subspaces) const -{ - return typify_invoke<1,TypifyCellType,CreateDefaultValueBuilderBase>(type.cell_type(), type, num_mapped_dims, subspace_size, expected_subspaces); -} - -//----------------------------------------------------------------------------- - -} diff --git a/eval/src/vespa/eval/tensor/default_value_builder_factory.h b/eval/src/vespa/eval/tensor/default_value_builder_factory.h deleted file mode 100644 index 67b1391ed78..00000000000 --- a/eval/src/vespa/eval/tensor/default_value_builder_factory.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/value_type.h> - -namespace vespalib::tensor { - -/** - * A factory that can generate ValueBuilder - * objects appropriate for the requested type. - */ -struct DefaultValueBuilderFactory : eval::ValueBuilderFactory { -private: - DefaultValueBuilderFactory(); - static DefaultValueBuilderFactory _factory; - ~DefaultValueBuilderFactory() override {} -protected: - std::unique_ptr<eval::ValueBuilderBase> create_value_builder_base(const eval::ValueType &type, - size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; -public: - static const DefaultValueBuilderFactory &get() { return _factory; } -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp index 0005a56736d..95d90a02a9e 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp @@ -83,7 +83,7 @@ void my_compiled_lambda_op(eval::InterpretedFunction::State &state, uint64_t par *bind_next++ = state.params->resolve(binding, state.stash).as_double(); } auto fun = params.token->get().get_function(); - ArrayRef<CT> dst_cells = state.stash.create_array<CT>(params.num_cells); + ArrayRef<CT> dst_cells = state.stash.create_uninitialized_array<CT>(params.num_cells); CT *dst = &dst_cells[0]; do { *dst++ = fun(&args[0]); @@ -119,7 +119,7 @@ void my_interpreted_lambda_op(eval::InterpretedFunction::State &state, uint64_t std::vector<double> labels(params.result_type.dimensions().size(), 0.0); ParamProxy param_proxy(labels, *state.params, params.bindings); InterpretedFunction::Context ctx(params.fun); - ArrayRef<CT> dst_cells = state.stash.create_array<CT>(params.num_cells); + ArrayRef<CT> dst_cells = state.stash.create_uninitialized_array<CT>(params.num_cells); CT *dst = &dst_cells[0]; do { *dst++ = params.fun.eval(ctx, param_proxy).as_double(); diff --git a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp index d6995256411..c41743200da 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp @@ -11,6 +11,7 @@ namespace vespalib::tensor { using vespalib::ArrayRef; +using eval::CellType; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -34,7 +35,7 @@ ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { if (inplace) { return unconstify(src_cells); } else { - return stash.create_array<CT>(src_cells.size()); + return stash.create_uninitialized_array<CT>(src_cells.size()); } } @@ -66,7 +67,7 @@ using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; bool is_dense(const TensorFunction &tf) { return tf.result_type().is_dense(); } bool is_double(const TensorFunction &tf) { return tf.result_type().is_double(); } -ValueType::CellType cell_type(const TensorFunction &tf) { return tf.result_type().cell_type(); } +CellType cell_type(const TensorFunction &tf) { return tf.result_type().cell_type(); } } // namespace vespalib::tensor::<unnamed> diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp index 5aca3799258..f492d12f05a 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -14,6 +14,7 @@ namespace vespalib::tensor { using vespalib::ArrayRef; +using eval::CellType; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -58,7 +59,7 @@ ArrayRef<OCT> make_dst_cells(ConstArrayRef<PCT> pri_cells, Stash &stash) { if constexpr (pri_mut && std::is_same<PCT,OCT>::value) { return unconstify(pri_cells); } else { - return stash.create_array<OCT>(pri_cells.size()); + return stash.create_uninitialized_array<OCT>(pri_cells.size()); } } @@ -106,11 +107,11 @@ using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool,TypifyOverlap>; //----------------------------------------------------------------------------- -bool can_use_as_output(const TensorFunction &fun, ValueType::CellType result_cell_type) { +bool can_use_as_output(const TensorFunction &fun, CellType result_cell_type) { return (fun.result_is_mutable() && (fun.result_type().cell_type() == result_cell_type)); } -Primary select_primary(const TensorFunction &lhs, const TensorFunction &rhs, ValueType::CellType result_cell_type) { +Primary select_primary(const TensorFunction &lhs, const TensorFunction &rhs, CellType result_cell_type) { size_t lhs_size = lhs.result_type().dense_subspace_size(); size_t rhs_size = rhs.result_type().dense_subspace_size(); if (lhs_size > rhs_size) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp index 1086af91ec4..d3297b335d3 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp @@ -32,7 +32,7 @@ ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { if (inplace) { return unconstify(src_cells); } else { - return stash.create_array<CT>(src_cells.size()); + return stash.create_uninitialized_array<CT>(src_cells.size()); } } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp index 1ad6e00f279..4b7f4936815 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp @@ -23,7 +23,7 @@ template <typename CT> void my_tensor_create_op(eval::InterpretedFunction::State &state, uint64_t param) { const auto &self = unwrap_param<DenseTensorCreateFunction::Self>(param); size_t pending_cells = self.result_size; - ArrayRef<CT> cells = state.stash.create_array<CT>(pending_cells); + ArrayRef<CT> cells = state.stash.create_uninitialized_array<CT>(pending_cells); while (pending_cells-- > 0) { cells[pending_cells] = (CT) state.peek(0).as_double(); state.stack.pop_back(); @@ -85,7 +85,7 @@ DenseTensorCreateFunction::optimize(const eval::TensorFunction &expr, Stash &sta const auto &zero_value = stash.create<DoubleValue>(0.0); const auto &zero_node = const_value(zero_value, stash); std::vector<Child> children(num_cells, zero_node); - for (const auto &cell: create->spec()) { + for (const auto &cell: create->map()) { size_t cell_idx = get_index(cell.first, expr.result_type()); children[cell_idx] = cell.second; } diff --git a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp index 5db533a4655..c49809f265f 100644 --- a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp +++ b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp @@ -19,6 +19,7 @@ LOG_SETUP(".eval.onnx_wrapper"); using vespalib::ArrayRef; using vespalib::ConstArrayRef; +using vespalib::eval::CellType; using vespalib::eval::ValueType; using vespalib::eval::TypifyCellType; @@ -110,18 +111,18 @@ auto convert_optimize(Onnx::Optimize optimize) { abort(); } -ValueType::CellType to_cell_type(Onnx::ElementType type) { +CellType to_cell_type(Onnx::ElementType type) { switch (type) { case Onnx::ElementType::INT8: [[fallthrough]]; case Onnx::ElementType::INT16: [[fallthrough]]; case Onnx::ElementType::UINT8: [[fallthrough]]; case Onnx::ElementType::UINT16: [[fallthrough]]; - case Onnx::ElementType::FLOAT: return ValueType::CellType::FLOAT; + case Onnx::ElementType::FLOAT: return CellType::FLOAT; case Onnx::ElementType::INT32: [[fallthrough]]; case Onnx::ElementType::INT64: [[fallthrough]]; case Onnx::ElementType::UINT32: [[fallthrough]]; case Onnx::ElementType::UINT64: [[fallthrough]]; - case Onnx::ElementType::DOUBLE: return ValueType::CellType::DOUBLE; + case Onnx::ElementType::DOUBLE: return CellType::DOUBLE; } abort(); } @@ -381,21 +382,21 @@ Onnx::EvalContext::convert_result(EvalContext &self, size_t idx) struct Onnx::EvalContext::SelectAdaptParam { template <typename ...Ts> static auto invoke() { return adapt_param<Ts...>; } - auto operator()(eval::ValueType::CellType ct) { + auto operator()(eval::CellType ct) { return typify_invoke<1,MyTypify,SelectAdaptParam>(ct); } }; struct Onnx::EvalContext::SelectConvertParam { template <typename ...Ts> static auto invoke() { return convert_param<Ts...>; } - auto operator()(eval::ValueType::CellType ct, Onnx::ElementType et) { + auto operator()(eval::CellType ct, Onnx::ElementType et) { return typify_invoke<2,MyTypify,SelectConvertParam>(ct, et); } }; struct Onnx::EvalContext::SelectConvertResult { template <typename ...Ts> static auto invoke() { return convert_result<Ts...>; } - auto operator()(Onnx::ElementType et, eval::ValueType::CellType ct) { + auto operator()(Onnx::ElementType et, eval::CellType ct) { return typify_invoke<2,MyTypify,SelectConvertResult>(et, ct); } }; diff --git a/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h b/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h index 87b1a5b47ed..7a35966cef8 100644 --- a/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h +++ b/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h @@ -6,7 +6,7 @@ namespace vespalib::tensor { -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; using TypedCells = vespalib::eval::TypedCells; template <typename TGT, typename... Args> @@ -30,7 +30,7 @@ decltype(auto) dispatch_2(A1 &&a, const TypedCells &b, Args &&...args) { struct GetCell { template<typename T> static double call(ConstArrayRef<T> arr, size_t idx) { - return arr[idx]; + return arr[idx]; } static double from(TypedCells src, size_t idx) { return dispatch_1<GetCell>(src, idx); diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp index 3c006fe64a0..ac91d073fd7 100644 --- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp @@ -20,7 +20,7 @@ struct CallVectorFromDoubles { template <typename CT> static TypedCells invoke(eval::InterpretedFunction::State &state, size_t numCells) { - ArrayRef<CT> outputCells = state.stash.create_array<CT>(numCells); + ArrayRef<CT> outputCells = state.stash.create_uninitialized_array<CT>(numCells); for (size_t i = numCells; i-- > 0; ) { outputCells[i] = (CT) state.peek(0).as_double(); state.stack.pop_back(); diff --git a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt deleted file mode 100644 index ceded3a7380..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_library(eval_tensor_mixed OBJECT - SOURCES - packed_labels.cpp - packed_mappings.cpp - packed_mappings_builder.cpp - packed_mixed_tensor_builder_factory.cpp - packed_mixed_tensor.cpp - packed_mixed_tensor_builder.cpp -) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp deleted file mode 100644 index 07c07bbb989..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_labels.h" -#include <assert.h> - -namespace vespalib::eval::packed_mixed_tensor { - -int32_t -PackedLabels::find_label(vespalib::stringref to_find) const -{ - uint32_t lo = 0; - uint32_t hi = num_labels(); - while (lo < hi) { - uint32_t mid = (lo + hi) / 2; - if (get_label(mid) < to_find) { - lo = mid + 1; - } else { - hi = mid; - } - } - assert(lo == hi); - if (lo < num_labels() && get_label(lo) == to_find) { - return lo; - } - return -1; -} - -vespalib::stringref -PackedLabels::get_label(uint32_t index) const -{ - assert(index < num_labels()); - - uint32_t this_offset = _offsets[index]; - uint32_t next_offset = _offsets[index+1]; - auto p = &_label_store[this_offset]; - size_t sz = next_offset - this_offset - 1; - return vespalib::stringref(p, sz); -} - -MemoryUsage -PackedLabels::extra_memory_usage() const -{ - MemoryUsage extra_usage; - size_t offsets_size = _offsets.size() * sizeof(uint32_t); - size_t labels_size = _label_store.size() * sizeof(char); - extra_usage.merge(MemoryUsage(offsets_size, offsets_size, 0, 0)); - extra_usage.merge(MemoryUsage(labels_size, labels_size, 0, 0)); - return extra_usage; -} - -void -PackedLabels::validate_labels(uint32_t num_labels) -{ - assert(num_labels == _offsets.size()-1); - for (uint32_t i = 0; i < num_labels; ++i) { - assert(_offsets[i] < _offsets[i+1]); - uint32_t last_byte_index = _offsets[i+1] - 1; - assert(last_byte_index < _label_store.size()); - assert(_label_store[last_byte_index] == 0); - } - assert(_label_store.size() == _offsets[num_labels]); - for (uint32_t i = 0; i+1 < num_labels; ++i) { - assert(get_label(i) < get_label(i+1)); - } -} - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.h b/eval/src/vespa/eval/tensor/mixed/packed_labels.h deleted file mode 100644 index 86024708568..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_labels.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/memory_usage_stuff.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/arrayref.h> - -namespace vespalib::eval::packed_mixed_tensor { - -/** - * Stores labels for sparse (mapped) tensor dimensions, - * where each unique label value is stored only once, - * and the values are sorted. References data that - * must be constant and owned by some object with - * enclosing lifetime. - **/ -class PackedLabels { -public: - PackedLabels(uint32_t num_labels, - ConstArrayRef<uint32_t> offsets, - ConstArrayRef<char> label_store) - : _offsets(offsets), - _label_store(label_store) - { - validate_labels(num_labels); - } - - uint32_t num_labels() const { return _offsets.size() - 1; } - - // returns -1 if the given label value cannot be found - int32_t find_label(vespalib::stringref value) const; - - vespalib::stringref get_label(uint32_t index) const; - - MemoryUsage extra_memory_usage() const; -private: - const ConstArrayRef<uint32_t> _offsets; - const ConstArrayRef<char> _label_store; - - void validate_labels(uint32_t num_labels); -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp deleted file mode 100644 index 9a3112f2a4f..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_mappings.h" -#include <assert.h> - -namespace vespalib::eval::packed_mixed_tensor { - - -int32_t -PackedMappings::subspace_of_address(const Address &address) const -{ - int32_t idx = sortid_of_address(address); - if (idx < 0) { - return -1; - } - uint32_t internal_idx = idx; - assert (internal_idx < _num_mappings); - return subspace_of_sortid(internal_idx); -} - -int32_t -PackedMappings::subspace_of_enums(const InternalAddress &address) const -{ - int32_t idx = sortid_of_enums(address); - if (idx < 0) { - return -1; - } - uint32_t internal_idx = idx; - assert (internal_idx < _num_mappings); - return subspace_of_sortid(internal_idx); -} - -int32_t -PackedMappings::sortid_of_address(const Address &address) const -{ - if (_num_dims == 0) return 0; - assert(address.size() == _num_dims); - std::vector<uint32_t> to_find; - to_find.reserve(_num_dims); - for (const auto & label_value : address) { - int32_t label_idx = _label_store.find_label(label_value); - if (label_idx < 0) { - return -1; - } - to_find.push_back(label_idx); - } - return sortid_of_enums(to_find); -} - -int32_t -PackedMappings::sortid_of_enums(const InternalAddress &address) const -{ - if (_num_dims == 0) return 0; - assert(address.size() == _num_dims); - const uint32_t * to_find = &address[0]; - uint32_t lo = 0; - uint32_t hi = _num_mappings; - while (lo < hi) { - uint32_t mid = (lo + hi) / 2; - if (enums_compare(ptr_of_sortid(mid), to_find) < 0) { - lo = mid + 1; - } else { - hi = mid; - } - } - assert(lo == hi); - if ((lo < _num_mappings) && - (enums_compare(ptr_of_sortid(lo), to_find) == 0)) - { - return lo; - } - return -1; -} - -/** returns subspace_index */ -uint32_t -PackedMappings::fill_enums_by_sortid(uint32_t internal_index, InternalAddress &address) const -{ - assert(internal_index < _num_mappings); - uint32_t offset = offset_of_mapping_data(internal_index); - address.resize(_num_dims); - for (uint32_t i = 0; i < _num_dims; ++i) { - address[i] = _int_store[offset++]; - } - return _int_store[offset]; -} - -/** returns subspace_index */ -uint32_t -PackedMappings::fill_address_by_sortid(uint32_t internal_index, Address &address) const -{ - assert(internal_index < _num_mappings); - uint32_t offset = offset_of_mapping_data(internal_index); - address.resize(_num_dims); - for (uint32_t i = 0; i < _num_dims; ++i) { - uint32_t label_idx = _int_store[offset++]; - address[i] = _label_store.get_label(label_idx); - } - return _int_store[offset]; -} - -MemoryUsage -PackedMappings::extra_memory_usage() const -{ - MemoryUsage extra_usage; - size_t store_size = _int_store.size() * sizeof(uint32_t); - extra_usage.merge(MemoryUsage(store_size, store_size, 0, 0)); - extra_usage.merge(_label_store.extra_memory_usage()); - return extra_usage; -} - -void -PackedMappings::validate() const -{ - assert((_num_mappings * (1 + _num_dims)) == _int_store.size()); - auto iter = _int_store.cbegin(); - std::vector<uint32_t> prev; - std::vector<uint32_t> next; - for (uint32_t i = 0; i < _num_mappings; ++i) { - next.clear(); - for (uint32_t j = 0; j < _num_dims; ++j) { - uint32_t label_index = *iter++; - next.push_back(label_index); - assert(label_index < _label_store.num_labels()); - } - if (_num_dims == 0) { - assert(next == prev); - assert(i == 0); - assert(_num_mappings == 1); - } else { - assert(prev < next); - } - std::swap(prev, next); - uint32_t subspace_index = *iter++; - assert(subspace_index < _num_mappings); - } - assert(iter == _int_store.cend()); -} - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h deleted file mode 100644 index 2c9f4b04a00..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "packed_labels.h" -#include <vector> - -namespace vespalib::eval::packed_mixed_tensor { - -/** - * Mappings for sparse tensor dimensions. - * - * Each address (conceptually "array of string") - * maps to a "subspace" (currently in the - * order that addresses were added to a builder). - * - * Internally addresses are lexicographically - * sorted, and you can iterate over them in sort - * order with the fill_*() methods. - * - * (Note: we may want to change this so subspaces - * are always sorted by address, so the "subspace" - * index and the "sortid" index become equivalent). - * - * Allows using the internal label enumerations - * instead of working with strings all the time. - * - * NOTE: Making a copy of PackedMappings will not copy - * the underlying data, these must then stay alive - * and unchanged for the lifetime of the copy as well. - **/ -class PackedMappings { -public: - using Address = std::vector<vespalib::stringref>; - using InternalAddress = std::vector<uint32_t>; - - uint32_t size() const { return _num_mappings; } - uint32_t num_mapped_dims() const { return _num_dims; } - - // returns -1 if mapping does not contain address - int32_t subspace_of_enums(const InternalAddress &address) const; - int32_t subspace_of_address(const Address &address) const; - - /** returns "subspace" index */ - uint32_t fill_address_by_sortid(uint32_t sortid, Address &address) const; - uint32_t fill_enums_by_sortid(uint32_t sortid, InternalAddress &address) const; - - // mapping from label enum to stringref (and vice versa) - const PackedLabels & label_store() const { return _label_store; } - - MemoryUsage extra_memory_usage() const; - -private: - PackedMappings(uint32_t num_dims, uint32_t num_mappings, - ConstArrayRef<uint32_t> int_store, - PackedLabels label_store) - : _num_dims(num_dims), - _num_mappings(num_mappings), - _int_store(int_store), - _label_store(label_store) - { - validate(); - } - friend class PackedMappingsBuilder; - - void validate() const; - - const uint32_t _num_dims; - const uint32_t _num_mappings; - /* - _int_store contains data corresponding to this model: - struct IntStore { - // sorted lexicographically by label_enums: - struct MappingData { - uint32_t label_enums[num_dims]; - uint32_t subspace_index; - } mappings[num_mappings]; - }; - */ - const ConstArrayRef<uint32_t> _int_store; - const PackedLabels _label_store; - - int enums_compare(const uint32_t *a, const uint32_t *b) const { - for (size_t i = 0; i < _num_dims; ++i) { - if (a[i] < b[i]) return -1; - if (a[i] > b[i]) return 1; - } - return 0; - } - - uint32_t offset_of_mapping_data(uint32_t idx) const { - return (idx * (1 + _num_dims)); - } - uint32_t subspace_of_sortid(uint32_t internal_index) const { - uint32_t offset = offset_of_mapping_data(internal_index); - return _int_store[offset + _num_dims]; - } - const uint32_t * ptr_of_sortid(uint32_t internal_index) const { - return &_int_store[offset_of_mapping_data(internal_index)]; - } - - int32_t sortid_of_address(const Address &address) const; - int32_t sortid_of_enums(const InternalAddress &address) const; -public: - static void operator delete(void *ptr, size_t sz) { - if (sz != sizeof(PackedMappings)) { - abort(); - } - size_t extra = ((const PackedMappings *)ptr)->extra_memory_usage().usedBytes(); - ::operator delete(ptr, sz + extra); - } -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp deleted file mode 100644 index abf142f9650..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_mappings_builder.h" -#include <assert.h> - -namespace vespalib::eval::packed_mixed_tensor { - -PackedMappingsBuilder::~PackedMappingsBuilder() = default; - -uint32_t -PackedMappingsBuilder::add_mapping_for(ConstArrayRef<vespalib::stringref> address_in) -{ - SparseAddress address; - for (auto & label_value : address_in) { - // store label string in our own set: - auto iter = _labels.insert(label_value).first; - address.push_back(*iter); - } - assert(address.size() == _num_dims); - uint32_t next_index = _mappings.size(); - auto iter = _mappings.emplace(address, next_index).first; - return iter->second; -} - - -size_t -PackedMappingsBuilder::extra_memory() const -{ - size_t int_store_cnt = (1 + _num_dims) * _mappings.size(); - size_t int_store_size = int_store_cnt * sizeof(uint32_t); - size_t label_cnt = _labels.size(); - size_t label_offsets_size = (1 + label_cnt) * sizeof(uint32_t); - size_t label_bytes = 0; - for (const auto & label_value : _labels) { - label_bytes += (label_value.size() + 1); - } - size_t extra_size = int_store_size + label_offsets_size + label_bytes; - return extra_size; -} - -PackedMappings -PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const -{ - size_t int_store_cnt = (1 + _num_dims) * _mappings.size(); - size_t int_store_size = int_store_cnt * sizeof(uint32_t); - size_t label_cnt = _labels.size(); - size_t label_offsets_size = (1 + label_cnt) * sizeof(uint32_t); - - size_t label_bytes = 0; - for (const auto & label_value : _labels) { - label_bytes += (label_value.size() + 1); - } - - ssize_t needs_sz = int_store_size + label_offsets_size + label_bytes; - ssize_t avail_sz = mem_end - mem_start; - assert(needs_sz <= avail_sz); - - uint32_t * int_store_mem = (uint32_t *) (void *) mem_start; - uint32_t * offsets_mem = (uint32_t *) (void *) (mem_start + int_store_size); - char * labels_mem = mem_start + int_store_size + label_offsets_size; - - ArrayRef<uint32_t> int_store_data(int_store_mem, int_store_cnt); - ArrayRef<uint32_t> label_offsets(offsets_mem, 1 + label_cnt); - ArrayRef<char> labels_data(labels_mem, label_bytes); - assert(labels_data.end() <= mem_end); - - size_t byte_idx = 0; - size_t label_num = 0; - for (const auto & label_value : _labels) { - label_offsets[label_num++] = byte_idx; - size_t len_with_zero = label_value.size() + 1; - memcpy(&labels_data[byte_idx], label_value.data(), len_with_zero); - byte_idx += len_with_zero; - } - assert(label_num == label_cnt); - label_offsets[label_num] = byte_idx; - - assert(labels_data.begin() + byte_idx == labels_data.end()); - - PackedLabels stored_labels(label_cnt, label_offsets, labels_data); - - size_t int_store_offset = 0; - for (const auto & kv : _mappings) { - const SparseAddress & k = kv.first; - uint32_t v = kv.second; - for (const auto & label_value : k) { - int32_t label_idx = stored_labels.find_label(label_value); - assert(label_idx >= 0); - assert(uint32_t(label_idx) < label_num); - int_store_data[int_store_offset++] = label_idx; - } - int_store_data[int_store_offset++] = v; - } - assert(int_store_offset == int_store_cnt); - - return PackedMappings(_num_dims, _mappings.size(), - int_store_data, stored_labels); -} - -std::unique_ptr<PackedMappings> -PackedMappingsBuilder::build_mappings() const -{ - size_t self_size = sizeof(PackedMappings); - size_t total_size = self_size + extra_memory(); - - char * mem = (char *) operator new(total_size); - auto self_data = target_memory(mem + self_size, mem + total_size); - - PackedMappings * built = new (mem) PackedMappings(self_data); - - return std::unique_ptr<PackedMappings>(built); -} - -} // namespace - - - diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h deleted file mode 100644 index 01f16a8b8e1..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "packed_mappings.h" -#include <vespa/vespalib/stllike/string.h> -#include <map> -#include <memory> -#include <set> -#include <vector> - -namespace vespalib::eval::packed_mixed_tensor { - -/** - * Builder for PackedMappings. - * Copies label values in all addresses added - * and packs all resulting data into a block of memory - * held by the built object, usually part of a larger - * aggregating object by using target_memory() method. - **/ -class PackedMappingsBuilder { -public: - PackedMappingsBuilder(uint32_t num_mapped_dims) - : _num_dims(num_mapped_dims), - _labels(), - _mappings() - {} - - ~PackedMappingsBuilder(); - - // returns a new index for new addresses - // may be called multiple times with same address, - // will then return the same index for that address. - uint32_t add_mapping_for(ConstArrayRef<vespalib::stringref> address); - - // how much extra memory is needed by target_memory - // not including sizeof(PackedMappings) - size_t extra_memory() const; - - // put data that PackedMappings can refer to in the given - // memory block, and return an object referring to it. - PackedMappings target_memory(char *mem_start, char *mem_end) const; - - // number of dimensions - uint32_t num_mapped_dims() const { return _num_dims; } - - // how many unique addresses have been added? - size_t size() const { return _mappings.size(); } - - // build a self-contained PackedMappings object; - // used for unit testing. - std::unique_ptr<PackedMappings> build_mappings() const; - -private: - uint32_t _num_dims; - std::set<vespalib::string> _labels; - using SparseAddress = std::vector<vespalib::stringref>; - using IndexMap = std::map<SparseAddress, uint32_t>; - IndexMap _mappings; -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp deleted file mode 100644 index e22828cbf0c..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_mixed_tensor.h" -#include <vespa/vespalib/util/typify.h> - -namespace vespalib::eval::packed_mixed_tensor { - -namespace { - -struct MySize { - template <typename CT> static size_t invoke(TypedCells cells) { - return (sizeof(CT) * cells.size); - } -}; - -} // namespace <unnamed> - -/*********************************************************************************/ - -class PackedMixedTensorIndexView : public Value::Index::View -{ -private: - const PackedMappings& _mappings; - const std::vector<size_t> _view_dims; - std::vector<uint32_t> _lookup_enums; - std::vector<uint32_t> _full_enums; - size_t _index; - - size_t num_full_dims() const { return _mappings.num_mapped_dims(); } - size_t num_view_dims() const { return _view_dims.size(); } - size_t num_rest_dims() const { return num_full_dims() - num_view_dims(); } -public: - PackedMixedTensorIndexView(const PackedMappings& mappings, - const std::vector<size_t> &dims) - : _mappings(mappings), - _view_dims(dims), - _lookup_enums(), - _index(0) - { - _lookup_enums.reserve(num_view_dims()); - _full_enums.resize(num_full_dims()); - } - - void lookup(ConstArrayRef<const vespalib::stringref*> addr) override; - bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override; - ~PackedMixedTensorIndexView() override = default; -}; - -void -PackedMixedTensorIndexView::lookup(ConstArrayRef<const vespalib::stringref*> addr) -{ - _index = 0; - assert(addr.size() == num_view_dims()); - _lookup_enums.clear(); - for (const vespalib::stringref * label_ptr : addr) { - int32_t label_enum = _mappings.label_store().find_label(*label_ptr); - if (label_enum < 0) { - // cannot match - _index = _mappings.size(); - break; - } - _lookup_enums.push_back(label_enum); - } -} - -bool -PackedMixedTensorIndexView::next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) -{ - assert(addr_out.size() == num_rest_dims()); - while (_index < _mappings.size()) { - idx_out = _mappings.fill_enums_by_sortid(_index++, _full_enums); - bool couldmatch = true; - size_t vd_idx = 0; - size_t ao_idx = 0; - for (size_t i = 0; i < num_full_dims(); ++i) { - if (vd_idx < num_view_dims()) { - size_t next_view_dim = _view_dims[vd_idx]; - if (i == next_view_dim) { - if (_lookup_enums[vd_idx] == _full_enums[i]) { - // match in this dimension - ++vd_idx; - continue; - } else { - // does not match - couldmatch = false; - break; - } - } - } - // not a view dimension: - uint32_t label_enum = _full_enums[i]; - *addr_out[ao_idx] = _mappings.label_store().get_label(label_enum); - ++ao_idx; - } - if (couldmatch) { - assert(vd_idx == num_view_dims()); - assert(ao_idx == num_rest_dims()); - return true; - } - } - return false; -} - -/*********************************************************************************/ - -class PackedMixedTensorLookup : public Value::Index::View -{ -private: - const PackedMappings& _mappings; - std::vector<uint32_t> _lookup_enums; - bool _first_time; - - size_t num_full_dims() const { return _mappings.num_mapped_dims(); } -public: - PackedMixedTensorLookup(const PackedMappings& mappings) - : _mappings(mappings), - _lookup_enums(), - _first_time(false) - { - _lookup_enums.reserve(num_full_dims()); - } - - void lookup(ConstArrayRef<const vespalib::stringref*> addr) override; - bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override; - ~PackedMixedTensorLookup() override = default; -}; - -void -PackedMixedTensorLookup::lookup(ConstArrayRef<const vespalib::stringref*> addr) -{ - assert(addr.size() == num_full_dims()); - _lookup_enums.clear(); - for (const vespalib::stringref * label_ptr : addr) { - int32_t label_enum = _mappings.label_store().find_label(*label_ptr); - if (label_enum < 0) { - // cannot match - _first_time = false; - return; - } - _lookup_enums.push_back(label_enum); - } - _first_time = true; -} - -bool -PackedMixedTensorLookup::next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) -{ - assert(addr_out.size() == 0); - if (_first_time) { - _first_time = false; - int32_t subspace = _mappings.subspace_of_enums(_lookup_enums); - if (subspace >= 0) { - idx_out = subspace; - return true; - } - } - return false; -} - -/*********************************************************************************/ - -class PackedMixedTensorAllMappings : public Value::Index::View -{ -private: - const PackedMappings& _mappings; - std::vector<vespalib::stringref> _full_address; - size_t _index; - -public: - PackedMixedTensorAllMappings(const PackedMappings& mappings) - : _mappings(mappings), - _full_address(), - _index(0) - { - _full_address.resize(_mappings.num_mapped_dims()); - } - - void lookup(ConstArrayRef<const vespalib::stringref*> addr) override; - bool next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) override; - ~PackedMixedTensorAllMappings() override = default; -}; - -void -PackedMixedTensorAllMappings::lookup(ConstArrayRef<const vespalib::stringref*> addr) -{ - _index = 0; - assert(addr.size() == 0); -} - -bool -PackedMixedTensorAllMappings::next_result(ConstArrayRef<vespalib::stringref*> addr_out, size_t &idx_out) -{ - assert(addr_out.size() == _mappings.num_mapped_dims()); - while (_index < _mappings.size()) { - idx_out = _mappings.fill_address_by_sortid(_index++, _full_address); - for (size_t i = 0; i < _mappings.num_mapped_dims(); ++i) { - *addr_out[i] = _full_address[i]; - } - return true; - } - return false; -} - -/*********************************************************************************/ - -PackedMixedTensor::PackedMixedTensor(const ValueType &type, - TypedCells cells, - const PackedMappings &mappings) - : _type(type), - _cells(cells), - _mappings(mappings) -{ - assert(type.cell_type() == _cells.type); -} - -PackedMixedTensor::~PackedMixedTensor() = default; - -MemoryUsage -PackedMixedTensor::get_memory_usage() const { - MemoryUsage usage = self_memory_usage<PackedMixedTensor>(); - usage.merge(_mappings.extra_memory_usage()); - size_t plus = add_for_alignment(usage.usedBytes()); - usage.merge(MemoryUsage(plus, plus, 0, 0)); - size_t cells_size = typify_invoke<1,TypifyCellType,MySize>(_cells.type, _cells); - usage.merge(MemoryUsage(cells_size, cells_size, 0, 0)); - return usage; -} - -std::unique_ptr<Value::Index::View> -PackedMixedTensor::create_view(const std::vector<size_t> &dims) const -{ - if (dims.size() == 0) { - return std::make_unique<PackedMixedTensorAllMappings>(_mappings); - } - for (size_t i = 1; i < dims.size(); ++i) { - assert(dims[i-1] < dims[i]); - assert(dims[i] < _mappings.num_mapped_dims()); - } - if (dims.size() == _mappings.num_mapped_dims()) { - return std::make_unique<PackedMixedTensorLookup>(_mappings); - } - return std::make_unique<PackedMixedTensorIndexView>(_mappings, dims); -} - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h deleted file mode 100644 index 013d2d7e07c..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/value_type.h> -#include <vespa/eval/eval/simple_value.h> - -#include <vespa/eval/tensor/mixed/packed_mappings.h> -#include <vespa/eval/tensor/mixed/packed_mappings_builder.h> - -namespace vespalib::eval::packed_mixed_tensor { - -/** - * An implementation of Value modeling a mixed tensor, - * where all the data (cells and sparse address mappings) - * can reside in a self-contained, contigous block of memory. - * Currently must be built by a PackedMixedTensorBuilder. - * Immutable (all data always const). - **/ -class PackedMixedTensor : public Value, public Value::Index -{ -private: - const ValueType _type; - const TypedCells _cells; - const PackedMappings _mappings; - - PackedMixedTensor(const ValueType &type, - TypedCells cells, - const PackedMappings &mappings); - - template<typename T> friend class PackedMixedTensorBuilder; - -public: - ~PackedMixedTensor() override; - - // Value API: - const ValueType &type() const override { return _type; } - const Value::Index &index() const override { return *this; } - TypedCells cells() const override { return _cells; } - - MemoryUsage get_memory_usage() const override; - - // Value::Index API: - size_t size() const override { return _mappings.size(); } - std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; - - // memory management: - static size_t add_for_alignment(size_t sz) { - size_t unalign = sz & 15; - return (unalign == 0) ? unalign : (16 - unalign); - } - static void operator delete(void *ptr, size_t sz) { - if (sz != sizeof(PackedMixedTensor)) { - abort(); - } - size_t allocated_sz = ((PackedMixedTensor *)ptr)->get_memory_usage().usedBytes(); - ::operator delete(ptr, allocated_sz); - } -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp deleted file mode 100644 index ef08ed20a9d..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_mixed_tensor_builder.h" - -namespace vespalib::eval::packed_mixed_tensor { - -template <typename T> -ArrayRef<T> -PackedMixedTensorBuilder<T>::add_subspace(ConstArrayRef<vespalib::stringref> addr_in) -{ - std::vector<vespalib::stringref> addr(addr_in.begin(), addr_in.end()); - uint32_t idx = _mappings_builder.add_mapping_for(addr); - size_t offset = idx * _subspace_size; - assert(offset == _cells.size()); - _cells.resize(offset + _subspace_size); - return ArrayRef<T>(&_cells[offset], _subspace_size); -} - - -template <typename T> -std::unique_ptr<Value> -PackedMixedTensorBuilder<T>::build(std::unique_ptr<ValueBuilder<T>>) -{ - size_t self_size = sizeof(PackedMixedTensor); - size_t mappings_size = _mappings_builder.extra_memory(); - // align cells: - mappings_size += PackedMixedTensor::add_for_alignment(self_size + mappings_size); - size_t cells_size = sizeof(T) * _cells.size(); - size_t total_size = self_size + mappings_size + cells_size; - - char *mem = (char *) operator new(total_size); - char *mappings_mem = mem + self_size; - char *cells_mem = mappings_mem + mappings_size; - - // fill mapping data: - auto mappings = _mappings_builder.target_memory(mappings_mem, cells_mem); - - // copy cells: - memcpy(cells_mem, &_cells[0], cells_size); - ConstArrayRef<T> cells((T *)(void *) cells_mem, _cells.size()); - - PackedMixedTensor * built = - new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); - - return std::unique_ptr<PackedMixedTensor>(built); -} - -template class PackedMixedTensorBuilder<float>; -template class PackedMixedTensorBuilder<double>; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h deleted file mode 100644 index a683b82dd24..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "packed_mixed_tensor.h" - -namespace vespalib::eval::packed_mixed_tensor { - -/** - * A builder for PackedMixedTensor objects - * appropriate for cell type T. - **/ -template <typename T> -class PackedMixedTensorBuilder : public ValueBuilder<T> -{ -private: - const ValueType & _type; - size_t _subspace_size; - std::vector<T> _cells; - PackedMappingsBuilder _mappings_builder; -public: - PackedMixedTensorBuilder(const ValueType &type, - size_t num_mapped_in, - size_t subspace_size_in, - size_t expected_subspaces) - : _type(type), - _subspace_size(subspace_size_in), - _cells(), - _mappings_builder(num_mapped_in) - { - _cells.reserve(_subspace_size * expected_subspaces); - } - - ~PackedMixedTensorBuilder() override = default; - - ArrayRef<T> add_subspace(ConstArrayRef<vespalib::stringref> addr) override; - std::unique_ptr<Value> build(std::unique_ptr<ValueBuilder<T>> self) override; -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp deleted file mode 100644 index 48eedd86f7f..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packed_mixed_tensor_builder_factory.h" -#include "packed_mixed_tensor_builder.h" - -#include <vespa/vespalib/util/typify.h> - -namespace vespalib::eval { - -namespace { - -struct CreatePackedMixedTensorBuilder { - template <typename T, typename ...Args> - static std::unique_ptr<ValueBuilderBase> invoke(const ValueType &type, Args &&...args) - { - assert(check_cell_type<T>(type.cell_type())); - return std::make_unique<packed_mixed_tensor::PackedMixedTensorBuilder<T>>(type, std::forward<Args>(args)...); - } -}; - -} // namespace <unnamed> - -PackedMixedTensorBuilderFactory::PackedMixedTensorBuilderFactory() = default; -PackedMixedTensorBuilderFactory PackedMixedTensorBuilderFactory::_factory; - -std::unique_ptr<ValueBuilderBase> -PackedMixedTensorBuilderFactory::create_value_builder_base(const ValueType &type, - size_t num_mapped_in, - size_t subspace_size_in, - size_t expected_subspaces) const -{ - return typify_invoke<1,TypifyCellType,CreatePackedMixedTensorBuilder>(type.cell_type(), - type, num_mapped_in, subspace_size_in, expected_subspaces); -} - -} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h deleted file mode 100644 index 20a581e2b35..00000000000 --- a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/value_type.h> -#include <vespa/eval/eval/simple_value.h> - -namespace vespalib::eval { - -/** - * A factory that can generate PackedMixedTensorBuilder - * objects appropriate for the requested CellType. - */ -struct PackedMixedTensorBuilderFactory : ValueBuilderFactory { -private: - PackedMixedTensorBuilderFactory(); - static PackedMixedTensorBuilderFactory _factory; - ~PackedMixedTensorBuilderFactory() override {} -protected: - std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, - size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; -public: - static const PackedMixedTensorBuilderFactory &get() { return _factory; } -}; - -} // namespace diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index 13c4711668b..837b135c0aa 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -8,7 +8,7 @@ using vespalib::nbostream; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; namespace vespalib::tensor { diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h index 21618dcb6ce..f0516e9fcc9 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h @@ -18,7 +18,7 @@ class DenseTensorView; class DenseBinaryFormat { public: - using CellType = eval::ValueType::CellType; + using CellType = vespalib::eval::CellType; static void serialize(nbostream &stream, const DenseTensorView &tensor); static std::unique_ptr<DenseTensorView> deserialize(nbostream &stream, CellType cell_type); diff --git a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp index eda8f7eecc7..a4022c4f60a 100644 --- a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp @@ -12,8 +12,8 @@ #include <cassert> using vespalib::nbostream; +using vespalib::eval::CellType; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; namespace vespalib::tensor { diff --git a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h index 0611d7d5a23..d4c7fa4bf6f 100644 --- a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h @@ -17,7 +17,7 @@ class Tensor; class SparseBinaryFormat { public: - using CellType = eval::ValueType::CellType; + using CellType = eval::CellType; static void serialize(nbostream &stream, const Tensor &tensor); static std::unique_ptr<Tensor> deserialize(nbostream &stream, CellType cell_type); diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp index 758ceb43ab4..2d3d1f4a0ea 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp @@ -19,7 +19,7 @@ LOG_SETUP(".eval.tensor.serialization.typed_binary_format"); using vespalib::nbostream; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; namespace vespalib::tensor { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index cd60a082472..0ecf957d1d9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -330,12 +330,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag REGIONAL_CONTAINER_REGISTRY = defineFeatureFlag( - "regional-container-registry", - true, - "Whether host-admin should download images from the zone's regional container registry", - "Takes effect immediately"); - public static final UnboundBooleanFlag ENABLE_AUTOMATIC_REINDEXING = defineFeatureFlag( "enable-automatic-reindexing", false, @@ -343,6 +337,20 @@ public class Flags { "Takes effect on next internal redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag( + "use-power-of-two-choices-load-balancing", + false, + "Whether to use Power of two load balancing algorithm for application", + "Takes effect on next internal redeployment", + APPLICATION_ID); + + public static final UnboundBooleanFlag DYNAMIC_RECONFIGURATION_OF_ZOOKEEPER_CLUSTER = defineFeatureFlag( + "dynamic-reconfiguration-of-zookeeper-cluster", + false, + "Whether to allow dynamic reconfiguration of zookeeper cluster", + "Takes effect on next deployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index d074915a023..3a848b33c76 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -281,12 +281,12 @@ public abstract class ControllerHttpClient { if (response.statusCode() / 100 == 4) throw new IllegalArgumentException("Bad request for " + request + ": " + message); - throw new IOException("Failed " + request + ": " + message); + throw new IOException(message); } catch (IOException e) { // Catches the above, and timeout exceptions from the client. if (thrown == null) - thrown = new UncheckedIOException(e); + thrown = new UncheckedIOException("Failed " + request + ": " + e, e); else thrown.addSuppressed(e); diff --git a/http-utils/pom.xml b/http-utils/pom.xml index 6d2e009cf8c..1f85658430f 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -26,6 +26,11 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>provided</scope> + </dependency> <!-- compile scope --> <dependency> @@ -38,6 +43,17 @@ <artifactId>httpcore</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.httpcomponents.client5</groupId> + <artifactId>httpclient5</artifactId> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> <!-- test scope --> <dependency> diff --git a/http-utils/src/main/java/ai/vespa/util/http/VespaAsyncHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/VespaAsyncHttpClientBuilder.java new file mode 100644 index 00000000000..6c53ea0dc69 --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/VespaAsyncHttpClientBuilder.java @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http; + +import com.yahoo.security.tls.MixedMode; +import com.yahoo.security.tls.TlsContext; +import com.yahoo.security.tls.TransportSecurityUtils; +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.client5.http.impl.DefaultSchemePortResolver; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner; +import org.apache.hc.client5.http.nio.AsyncClientConnectionManager; +import org.apache.hc.client5.http.routing.HttpRoutePlanner; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.http.protocol.HttpContext; + +import javax.net.ssl.SSLParameters; + +/** + * Async http client builder for internal Vespa communications over http/https. + * Configures Vespa mTLS and handles TLS mixed mode automatically. + * Client should only be used for requests to Vespa services. + * + * Caveats: + * - custom connection manager must be configured through {@link #create(AsyncConnectionManagerFactory)}. + * + * @author bjorncs + */ +public class VespaAsyncHttpClientBuilder { + + public interface AsyncConnectionManagerFactory { + AsyncClientConnectionManager create(TlsStrategy tlsStrategy); + } + + public static HttpAsyncClientBuilder create() { + return create( + tlsStrategy -> PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build()); + } + + public static HttpAsyncClientBuilder create(AsyncConnectionManagerFactory factory) { + HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create(); + TlsContext vespaTlsContext = TransportSecurityUtils.createTlsContext().orElse(null); + TlsStrategy tlsStrategy; + if (vespaTlsContext != null) { + SSLParameters vespaTlsParameters = vespaTlsContext.parameters(); + tlsStrategy = ClientTlsStrategyBuilder.create() + .setHostnameVerifier(new NoopHostnameVerifier()) + .setSslContext(vespaTlsContext.context()) + .setTlsVersions(vespaTlsParameters.getProtocols()) + .setCiphers(vespaTlsParameters.getCipherSuites()) + .build(); + if (TransportSecurityUtils.getInsecureMixedMode() != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) { + clientBuilder.setRoutePlanner(new HttpToHttpsRoutePlanner()); + } + } else { + tlsStrategy = ClientTlsStrategyBuilder.create().build(); + } + clientBuilder.disableConnectionState(); // Share connections between subsequent requests + clientBuilder.disableCookieManagement(); + clientBuilder.disableAuthCaching(); + clientBuilder.disableRedirectHandling(); + clientBuilder.setConnectionManager(factory.create(tlsStrategy)); + clientBuilder.setConnectionManagerShared(false); + return clientBuilder; + } + + private static class HttpToHttpsRoutePlanner implements HttpRoutePlanner { + + private final DefaultRoutePlanner defaultPlanner = new DefaultRoutePlanner(new DefaultSchemePortResolver()); + + @Override + public HttpRoute determineRoute(HttpHost target, HttpContext context) throws HttpException { + HttpRoute originalRoute = defaultPlanner.determineRoute(target, context); + HttpHost originalHost = originalRoute.getTargetHost(); + String originalScheme = originalHost.getSchemeName(); + String rewrittenScheme = originalScheme.equalsIgnoreCase("http") ? "https" : originalScheme; + boolean rewrittenSecure = target.getSchemeName().equalsIgnoreCase("https"); + HttpHost rewrittenHost = new HttpHost( + rewrittenScheme, originalHost.getAddress(), originalHost.getHostName(), originalHost.getPort()); + return new HttpRoute( + rewrittenHost, + originalRoute.getLocalAddress(), + originalRoute.getProxyHost(), + rewrittenSecure, + originalRoute.getTunnelType(), + originalRoute.getLayerType()); + } + } + +} diff --git a/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java b/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java index 2bac7f66799..c6afa889041 100644 --- a/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java +++ b/jaxrs_client_utils/src/main/java/ai/vespa/util/http/VespaClientBuilderFactory.java @@ -27,8 +27,10 @@ import static java.util.logging.Level.CONFIG; * - hostname verification is not enabled - CN/SAN verification is assumed to be handled by the underlying x509 trust manager. * - ssl context or hostname verifier must not be overridden by the caller * + * @deprecated Use Apache httpclient based client factory instead (VespaHttpClientBuilder). * @author bjorncs */ +@Deprecated(forRemoval = true) public class VespaClientBuilderFactory implements AutoCloseable { private static final Logger log = Logger.getLogger(VespaClientBuilderFactory.class.getName()); diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/VespaJerseyJaxRsClientFactory.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/VespaJerseyJaxRsClientFactory.java index bdc89d737d4..6d1c1c71f21 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/VespaJerseyJaxRsClientFactory.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/VespaJerseyJaxRsClientFactory.java @@ -17,10 +17,13 @@ import java.util.List; /** * Factory for creating Jersey based Vespa clients from a JAX-RS resource interface. * + * @deprecated Use Apache httpclient based client factory instead (VespaHttpClientBuilder). * @author bjorncs */ +@Deprecated(forRemoval = true) public class VespaJerseyJaxRsClientFactory implements JaxRsClientFactory, AutoCloseable { + @SuppressWarnings("removal") private final VespaClientBuilderFactory clientBuilder = new VespaClientBuilderFactory(); // Client is a heavy-weight object with a finalizer so we create only one and re-use it private final Client client; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilter.java new file mode 100644 index 00000000000..b891212031f --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilter.java @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.jdisc.http.filter.security.misc; + +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; + +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Optional; + +public class VespaTlsFilter extends JsonSecurityRequestFilterBase { + + @Override + protected Optional<ErrorResponse> filter(DiscFilterRequest request) { + return request.getClientCertificateChain().isEmpty() + ? Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden to access this path")) + : Optional.empty(); + } +} diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java new file mode 100644 index 00000000000..294126eb349 --- /dev/null +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java @@ -0,0 +1,66 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.jdisc.http.filter.security.misc; + +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.net.URI; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.when; + +public class VespaTlsFilterTest { + + @Test + public void testFilter() { + assertSuccess(createRequest(List.of(createCertificate()))); + assertForbidden(createRequest(Collections.emptyList())); + } + + private static X509Certificate createCertificate() { + return X509CertificateBuilder + .fromKeypair( + KeyUtils.generateKeypair(KeyAlgorithm.EC), new X500Principal("CN=test"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), + SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(1)) + .build(); + } + + private static DiscFilterRequest createRequest(List<X509Certificate> certChain) { + DiscFilterRequest request = Mockito.mock(DiscFilterRequest.class); + when(request.getClientCertificateChain()).thenReturn(certChain); + when(request.getMethod()).thenReturn("GET"); + when(request.getUri()).thenReturn(URI.create("http://localhost:8080/")); + return request; + } + + private static void assertForbidden(DiscFilterRequest request) { + VespaTlsFilter filter = new VespaTlsFilter(); + RequestHandlerTestDriver.MockResponseHandler handler = new RequestHandlerTestDriver.MockResponseHandler(); + filter.filter(request, handler); + assertEquals(Response.Status.FORBIDDEN, handler.getStatus()); + } + + private static void assertSuccess(DiscFilterRequest request) { + VespaTlsFilter filter = new VespaTlsFilter(); + RequestHandlerTestDriver.MockResponseHandler handler = new RequestHandlerTestDriver.MockResponseHandler(); + filter.filter(request, handler); + assertNull(handler.getResponse()); + } +} diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index b140e66f28a..7333db96b91 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -152,6 +152,7 @@ jetty-servlet-${jetty.version}.jar, jetty-servlets-${jetty.version}.jar, jetty-util-${jetty.version}.jar, + jetty-util-ajax-${jetty.version}.jar, component-jar-with-dependencies.jar </discPreInstallBundle> </configuration> diff --git a/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp b/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp index 715ef597767..fda3075fc28 100644 --- a/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp +++ b/messagebus/src/tests/simpleprotocol/simpleprotocol.cpp @@ -1,14 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/ireplyhandler.h> -#include <vespa/messagebus/network/identity.h> #include <vespa/messagebus/routing/routingcontext.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/component/vtag.h> @@ -26,15 +23,20 @@ Test::Main() SimpleProtocol protocol; EXPECT_TRUE(protocol.getName() == "Simple"); + EXPECT_EQUAL(152u, sizeof(Result)); + EXPECT_EQUAL(136u, sizeof(Error)); + EXPECT_EQUAL(56u, sizeof(Routable)); { // test protocol IRoutingPolicy::UP bogus = protocol.createPolicy("bogus", ""); - EXPECT_TRUE(bogus.get() == 0); + EXPECT_FALSE(bogus); } TEST_FLUSH(); { // test SimpleMessage - Message::UP msg(new SimpleMessage("test")); + EXPECT_EQUAL(104u, sizeof(Message)); + EXPECT_EQUAL(184u, sizeof(SimpleMessage)); + auto msg = std::make_unique<SimpleMessage>("test"); EXPECT_TRUE(!msg->isReply()); EXPECT_TRUE(msg->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(msg->getType() == SimpleProtocol::MESSAGE); @@ -42,7 +44,7 @@ Test::Main() Blob b = protocol.encode(version, *msg); EXPECT_TRUE(b.size() > 0); Routable::UP tmp = protocol.decode(version, BlobRef(b)); - ASSERT_TRUE(tmp.get() != 0); + ASSERT_TRUE(tmp); EXPECT_TRUE(!tmp->isReply()); EXPECT_TRUE(tmp->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(tmp->getType() == SimpleProtocol::MESSAGE); @@ -51,7 +53,9 @@ Test::Main() TEST_FLUSH(); { // test SimpleReply - Reply::UP reply(new SimpleReply("reply")); + EXPECT_EQUAL(96u, sizeof(Reply)); + EXPECT_EQUAL(160u, sizeof(SimpleReply)); + auto reply = std::make_unique<SimpleReply>("reply"); EXPECT_TRUE(reply->isReply()); EXPECT_TRUE(reply->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(reply->getType() == SimpleProtocol::REPLY); @@ -59,7 +63,7 @@ Test::Main() Blob b = protocol.encode(version, *reply); EXPECT_TRUE(b.size() > 0); Routable::UP tmp = protocol.decode(version, BlobRef(b)); - ASSERT_TRUE(tmp.get() != 0); + ASSERT_TRUE(tmp); EXPECT_TRUE(tmp->isReply()); EXPECT_TRUE(tmp->getProtocol() == SimpleProtocol::NAME); EXPECT_TRUE(tmp->getType() == SimpleProtocol::REPLY); diff --git a/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp b/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp index 344d38c8263..f97fc6a0010 100644 --- a/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp +++ b/messagebus/src/tests/trace-roundtrip/trace-roundtrip.cpp @@ -120,6 +120,6 @@ Test::Main() .addChild("Server reply") .addChild("Proxy reply") .addChild("Client reply"); - EXPECT_TRUE(reply->getTrace().getRoot().encode() == t.encode()); + EXPECT_TRUE(reply->getTrace().encode() == t.encode()); TEST_DONE(); } diff --git a/messagebus/src/vespa/messagebus/error.cpp b/messagebus/src/vespa/messagebus/error.cpp index 85c9b92eb19..85eec140db5 100644 --- a/messagebus/src/vespa/messagebus/error.cpp +++ b/messagebus/src/vespa/messagebus/error.cpp @@ -3,7 +3,7 @@ #include "errorcode.h" #include <vespa/vespalib/util/stringfmt.h> -using vespalib::make_string; +using vespalib::make_string_short::fmt; namespace mbus { @@ -13,9 +13,9 @@ Error::Error() _service() { } -Error::~Error() {} +Error::~Error() = default; -Error::Error(uint32_t c, const string &m, const string &s) +Error::Error(uint32_t c, vespalib::stringref m, vespalib::stringref s) : _code(c), _msg(m), _service(s) @@ -26,9 +26,9 @@ Error::toString() const { string name(ErrorCode::getName(_code)); if (name.empty()) { - name = make_string("%u", _code); + name = fmt("%u", _code); } - return make_string("[%s @ %s]: %s", name.c_str(), _service.empty() ? "localhost" : _service.c_str(), _msg.c_str()); + return fmt("[%s @ %s]: %s", name.c_str(), _service.empty() ? "localhost" : _service.c_str(), _msg.c_str()); } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/error.h b/messagebus/src/vespa/messagebus/error.h index 638463e3665..74936962113 100644 --- a/messagebus/src/vespa/messagebus/error.h +++ b/messagebus/src/vespa/messagebus/error.h @@ -36,7 +36,7 @@ public: * @param m error message * @param s error service **/ - Error(uint32_t c, const string &m, const string &s = ""); + Error(uint32_t c, vespalib::stringref m, vespalib::stringref s = ""); /** * Obtain the error code of this error. diff --git a/messagebus/src/vespa/messagebus/message.cpp b/messagebus/src/vespa/messagebus/message.cpp index 305ffb06aa0..8d17fd6b0ff 100644 --- a/messagebus/src/vespa/messagebus/message.cpp +++ b/messagebus/src/vespa/messagebus/message.cpp @@ -5,6 +5,7 @@ #include "ireplyhandler.h" #include "emptyreply.h" #include "errorcode.h" +#include "error.h" #include <vespa/vespalib/util/backtrace.h> #include <vespa/log/log.h> diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp index dca7f0c997f..86c9b139f1a 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -178,7 +178,7 @@ RPCSend::doRequestDone(FRT_RPCRequest *req) { } } else { FRT_Values &ret = *req->GetReturn(); - reply = createReply(ret, serviceName, error, trace.getRoot()); + reply = createReply(ret, serviceName, error, trace); } if (trace.shouldTrace(TraceLevel::SEND_RECEIVE)) { trace.trace(TraceLevel::SEND_RECEIVE, diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.h b/messagebus/src/vespa/messagebus/network/rpcsend.h index f3a9177d236..1ccdea6fbc5 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.h +++ b/messagebus/src/vespa/messagebus/network/rpcsend.h @@ -12,7 +12,7 @@ class FRT_ReflectionBuilder; namespace vespalib::slime { struct Cursor; } namespace vespalib { struct Memory; } -namespace vespalib { class TraceNode; } +namespace vespalib { class Trace; } namespace mbus { class Error; @@ -56,7 +56,7 @@ protected: virtual void build(FRT_ReflectionBuilder & builder) = 0; virtual std::unique_ptr<Reply> createReply(const FRT_Values & response, const string & serviceName, - Error & error, vespalib::TraceNode & rootTrace) const = 0; + Error & error, vespalib::Trace & trace) const = 0; virtual void encodeRequest(FRT_RPCRequest &req, const vespalib::Version &version, const Route & route, const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, const PayLoadFiller &filler, duration timeRemaining) const = 0; diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp index 388ab3309c4..775c90ea9ee 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp @@ -4,6 +4,7 @@ #include "rpcnetwork.h" #include "rpcserviceaddress.h" #include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/error.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/fnet/frt/reflection.h> @@ -115,7 +116,7 @@ RPCSendV1::toParams(const FRT_Values &args) const std::unique_ptr<Reply> -RPCSendV1::createReply(const FRT_Values & ret, const string & serviceName, Error & error, vespalib::TraceNode & rootTrace) const +RPCSendV1::createReply(const FRT_Values & ret, const string & serviceName, Error & error, vespalib::Trace & trace) const { vespalib::Version version = vespalib::Version(ret[0]._string._str); double retryDelay = ret[1]._double; @@ -127,7 +128,7 @@ RPCSendV1::createReply(const FRT_Values & ret, const string & serviceName, Error uint32_t errorServicesLen = ret[4]._string_array._len; const char *protocolName = ret[5]._string._str; BlobRef payload(ret[6]._data._buf, ret[6]._data._len); - const char *trace = ret[7]._string._str; + const char *traceStr = ret[7]._string._str; Reply::UP reply; if (payload.size() > 0) { @@ -141,7 +142,7 @@ RPCSendV1::createReply(const FRT_Values & ret, const string & serviceName, Error reply->addError(Error(errorCodes[i], errorMessages[i]._str, errorServices[i]._len > 0 ? errorServices[i]._str : serviceName.c_str())); } - rootTrace.addChild(TraceNode::decode(trace)); + trace.addChild(TraceNode::decode(traceStr)); return reply; } @@ -163,7 +164,7 @@ RPCSendV1::createResponse(FRT_Values & ret, const string & version, Reply & repl ret.AddString(reply.getProtocol().c_str()); ret.AddData(std::move(payload.payload()), payload.size()); if (reply.getTrace().getLevel() > 0) { - ret.AddString(reply.getTrace().getRoot().encode().c_str()); + ret.AddString(reply.getTrace().encode().c_str()); } else { ret.AddString(""); } diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv1.h b/messagebus/src/vespa/messagebus/network/rpcsendv1.h index 249acc50e0c..f9b4b0bdfe8 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv1.h +++ b/messagebus/src/vespa/messagebus/network/rpcsendv1.h @@ -17,7 +17,7 @@ private: const PayLoadFiller &filler, duration timeRemaining) const override; std::unique_ptr<Reply> createReply(const FRT_Values & response, const string & serviceName, - Error & error, vespalib::TraceNode & rootTrace) const override; + Error & error, vespalib::Trace & trace) const override; void createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const override; }; diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp index f7303ece20f..a2e3310046d 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp @@ -4,6 +4,7 @@ #include "rpcnetwork.h" #include "rpcserviceaddress.h" #include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/error.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/databuffer.h> @@ -188,7 +189,7 @@ RPCSendV2::toParams(const FRT_Values &args) const std::unique_ptr<Reply> RPCSendV2::createReply(const FRT_Values & ret, const string & serviceName, - Error & error, vespalib::TraceNode & rootTrace) const + Error & error, vespalib::Trace & rootTrace) const { uint8_t encoding = ret[3]._intval8; uint32_t uncompressedSize = ret[4]._intval32; @@ -240,7 +241,7 @@ RPCSendV2::createResponse(FRT_Values & ret, const string & version, Reply & repl root.setString(PROTOCOL_F, reply.getProtocol()); root.setData(BLOB_F, vespalib::Memory(payload.data(), payload.size())); if (reply.getTrace().getLevel() > 0) { - root.setString(TRACE_F, reply.getTrace().getRoot().encode()); + root.setString(TRACE_F, reply.getTrace().encode()); } if (reply.getNumErrors() > 0) { @@ -249,7 +250,7 @@ RPCSendV2::createResponse(FRT_Values & ret, const string & version, Reply & repl Cursor & error = array.addObject(); error.setLong(CODE_F, reply.getError(i).getCode()); error.setString(MSG_F, reply.getError(i).getMessage()); - error.setString(SERVICE_F, reply.getError(i).getService().c_str()); + error.setString(SERVICE_F, reply.getError(i).getService()); } } @@ -263,7 +264,6 @@ RPCSendV2::createResponse(FRT_Values & ret, const string & version, Reply & repl ret.AddInt32(toCompress.size()); assert(buf.getDataLen() <= INT32_MAX); ret.AddData(std::move(buf)); - } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.h b/messagebus/src/vespa/messagebus/network/rpcsendv2.h index c48aa90a9fb..da4154e70a8 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv2.h +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.h @@ -17,7 +17,7 @@ private: const PayLoadFiller &filler, duration timeRemaining) const override; std::unique_ptr<Reply> createReply(const FRT_Values & response, const string & serviceName, - Error & error, vespalib::TraceNode & rootTrace) const override; + Error & error, vespalib::Trace & trace) const override; void createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const override; }; diff --git a/messagebus/src/vespa/messagebus/reply.cpp b/messagebus/src/vespa/messagebus/reply.cpp index bf12698c86f..68bb7db1191 100644 --- a/messagebus/src/vespa/messagebus/reply.cpp +++ b/messagebus/src/vespa/messagebus/reply.cpp @@ -5,6 +5,7 @@ #include "ireplyhandler.h" #include "message.h" #include "tracelevel.h" +#include "error.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/backtrace.h> @@ -54,12 +55,6 @@ Reply::swapState(Routable &rhs) } } -bool -Reply::isReply() const -{ - return true; -} - void Reply::addError(const Error &e) { @@ -72,16 +67,25 @@ Reply::addError(const Error &e) bool Reply::hasFatalErrors() const { - for (std::vector<Error>::const_iterator it = _errors.begin(); - it != _errors.end(); ++it) + for (const Error & error : _errors) { - if (it->getCode() >= ErrorCode::FATAL_ERROR) { + if (error.getCode() >= ErrorCode::FATAL_ERROR) { return true; } } return false; } +const Error & +Reply::getError(uint32_t i) const { + return _errors[i]; +} + +uint32_t +Reply::getNumErrors() const { + return _errors.size(); +} + void Reply::setMessage(Message::UP msg) { _msg = std::move(msg); diff --git a/messagebus/src/vespa/messagebus/reply.h b/messagebus/src/vespa/messagebus/reply.h index 64b9f5c0b13..72493ec0ae6 100644 --- a/messagebus/src/vespa/messagebus/reply.h +++ b/messagebus/src/vespa/messagebus/reply.h @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "error.h" #include "routable.h" #include <vector> namespace mbus { class Message; +class Error; /** * A reply is a response to a message that has been sent throught the @@ -43,7 +43,7 @@ public: ~Reply() override; void swapState(Routable &rhs) override; - bool isReply() const override; + bool isReply() const override { return true; } /** * Add an Error to this Reply @@ -72,14 +72,14 @@ public: * @param i The index of the error to return. * @return The error at the given index. */ - const Error &getError(uint32_t i) const { return _errors[i]; } + const Error &getError(uint32_t i) const; /** * Returns the number of errors that this reply contains. * * @return The number of replies. */ - uint32_t getNumErrors() const { return _errors.size(); } + uint32_t getNumErrors() const; /** * Attach a Message to this Reply. If a Reply contains errors, messagebus diff --git a/messagebus/src/vespa/messagebus/result.cpp b/messagebus/src/vespa/messagebus/result.cpp index e99120d8c2d..970bb72e638 100644 --- a/messagebus/src/vespa/messagebus/result.cpp +++ b/messagebus/src/vespa/messagebus/result.cpp @@ -1,80 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "result.h" +#include "message.h" namespace mbus { -Result::Handover::Handover(bool a, const Error &e, Message *m) - : _accepted(a), - _error(e), - _msg(m) -{ } - Result::Result() : _accepted(true), _error(), _msg() { } -Result::Result(const Error &err, Message::UP msg) +Result::Result(const Error &err, std::unique_ptr<Message> msg) : _accepted(false), _error(err), _msg(std::move(msg)) { } -Result::Result(Result &&rhs) - : _accepted(rhs._accepted), - _error(rhs._error), - _msg(std::move(rhs._msg)) -{ } - -Result::Result(const Handover &rhs) - : _accepted(rhs._accepted), - _error(rhs._error), - _msg(rhs._msg) -{ } - -Result::~Result() {} - -bool -Result::isAccepted() const -{ - return _accepted; -} - -const Error & -Result::getError() const -{ - return _error; -} - -Message::UP -Result::getMessage() -{ - return std::move(_msg); -} - -Result::operator Handover() -{ - return Handover(_accepted, _error, _msg.release()); -} - -Result & -Result::operator=(Result &&rhs) -{ - _accepted = rhs._accepted; - _error = rhs._error; - _msg = std::move(rhs._msg); - return *this; -} - -Result & -Result::operator=(const Handover &rhs) -{ - _accepted = rhs._accepted; - _error = rhs._error; - _msg.reset(rhs._msg); - return *this; -} +Result::~Result() = default; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/result.h b/messagebus/src/vespa/messagebus/result.h index adb9c1ea81a..dcf6e4964e5 100644 --- a/messagebus/src/vespa/messagebus/result.h +++ b/messagebus/src/vespa/messagebus/result.h @@ -3,10 +3,11 @@ #pragma once #include "error.h" -#include "message.h" namespace mbus { +class Message; + /** * A Result object is used as return value when trying to send a * Message on a SourceSession. It says whether messagebus has accepted @@ -24,24 +25,10 @@ class Result private: bool _accepted; Error _error; - Message::UP _msg; + std::unique_ptr<Message> _msg; public: /** - * This inner class is used to implement destructive copy for - * return values. - **/ - class Handover { - friend class Result; - bool _accepted; - Error _error; - Message *_msg; - Handover(bool a, const Error &e, Message *m); - Handover(const Handover &); // not implemented - Handover &operator=(const Handover &); // not implemented - }; - - /** * Create a Result indicating that messagebus has accepted the * Message. **/ @@ -54,23 +41,11 @@ public: * @param err the reason for not accepting the Message * @param msg the message that did not get accepted **/ - Result(const Error &err, Message::UP msg); + Result(const Error &err, std::unique_ptr<Message> msg); - /** - * Move constructor - * - * @param rhs the original object - **/ - Result(Result &&rhs); - - /** - * Construct a new Result from an internal Handover object that - * has destructed the original Result. - * - * @param rhs handover object - **/ - Result(const Handover &rhs); + Result(Result &&rhs) = default; + Result &operator=(Result &&rhs) = default; ~Result(); /** @@ -78,14 +53,14 @@ public: * * @return true if the Message was accepted **/ - bool isAccepted() const; + bool isAccepted() const { return _accepted; } /** * Obtain the error causing the message not to be accepted. * * @return error **/ - const Error &getError() const; + const Error &getError() const { return _error; } /** * If the message was not accepted, this method may be used to get @@ -95,28 +70,7 @@ public: * * @return the Message that was not accepted **/ - Message::UP getMessage(); - - /** - * Perform an implicit typecast to support destructive copy of - * return values. - **/ - operator Handover(); - - /** - * Moving assignment operator - * - * @param rhs the original object - **/ - Result &operator=(Result &&rhs); - - /** - * Assign a Result from an internal Handover object that has - * destructed the original Result. - * - * @param rhs handover object - **/ - Result &operator=(const Handover &rhs); + std::unique_ptr<Message> getMessage() { return std::move(_msg); } }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/routable.h b/messagebus/src/vespa/messagebus/routable.h index 50cb4e090ce..2c83daded1e 100644 --- a/messagebus/src/vespa/messagebus/routable.h +++ b/messagebus/src/vespa/messagebus/routable.h @@ -24,9 +24,9 @@ namespace mbus { */ class Routable { private: - Context _context; - CallStack _stack; - Trace _trace; + Context _context; + CallStack _stack; + Trace _trace; public: /** @@ -90,6 +90,7 @@ public: * @return Trace object */ Trace &getTrace() { return _trace; } + Trace && steal_trace() { return std::move(_trace); } /** * Access the Trace object for this Routable. The Trace is part of the @@ -106,7 +107,7 @@ public: * * @param trace The trace to set. */ - void setTrace(const Trace &trace) { _trace = trace; } + void setTrace(Trace &&trace) { _trace = std::move(trace); } /** * Swaps the state that makes this routable unique to another routable. The diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.cpp b/messagebus/src/vespa/messagebus/routing/routingnode.cpp index f24afbc07ca..5a70f510dcc 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnode.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingnode.cpp @@ -184,10 +184,7 @@ RoutingNode::setReply(Reply::UP reply) { if (reply) { _shouldRetry = _resender != nullptr && _resender->shouldRetry(*reply); - if ( ! reply->getTrace().getRoot().isEmpty()) { - _trace.getRoot().addChild(std::move(reply->getTrace().getRoot())); - reply->getTrace().clear(); - } + _trace.addChild(reply->steal_trace()); } _reply = std::move(reply); } @@ -268,14 +265,12 @@ RoutingNode::notifyMerge() // Merges the trace information from all children into this. This method takes care not to spend cycles // manipulating the trace in case tracing is disabled. if (_trace.getLevel() > 0) { - TraceNode tail; + Trace tail; for (auto * child : _children) { - TraceNode &root = child->_trace.getRoot(); - tail.addChild(root); - root.clear(); + tail.addChild(std::move(child->_trace)); } tail.setStrict(false); - _trace.getRoot().addChild(tail); + _trace.addChild(std::move(tail)); } // Execute the {@link RoutingPolicy#merge(RoutingContext)} method of the current routing policy. If a diff --git a/messagebus/src/vespa/messagebus/sendproxy.cpp b/messagebus/src/vespa/messagebus/sendproxy.cpp index c884932664b..ff514f788cd 100644 --- a/messagebus/src/vespa/messagebus/sendproxy.cpp +++ b/messagebus/src/vespa/messagebus/sendproxy.cpp @@ -53,11 +53,10 @@ SendProxy::handleReply(Reply::UP reply) } else if (logger.wants(ns_log::Logger::spam)) { LOG(spam, "Trace for reply:\n%s", reply->getTrace().toString().c_str()); } - Trace empty; - trace.swap(empty); + trace.clear(); } else if (trace.getLevel() > 0) { - trace.getRoot().addChild(reply->getTrace().getRoot()); - trace.getRoot().normalize(); + trace.addChild(reply->steal_trace()); + trace.normalize(); } reply->swapState(*_msg); reply->setMessage(std::move(_msg)); diff --git a/messagebus_test/src/tests/trace/trace.cpp b/messagebus_test/src/tests/trace/trace.cpp index 334f00745da..1ab30303d2c 100644 --- a/messagebus_test/src/tests/trace/trace.cpp +++ b/messagebus_test/src/tests/trace/trace.cpp @@ -100,14 +100,14 @@ Test::Main() Reply::UP reply; SourceSession::UP ss = mb.getMessageBus().createSourceSession(src, SourceSessionParams()); for (int i = 0; i < 50; ++i) { - Message::UP msg(new SimpleMessage("test")); + auto msg = std::make_unique<SimpleMessage>("test"); msg->getTrace().setLevel(1); ss->send(std::move(msg), "test"); reply = src.getReply(10s); if (reply) { - reply->getTrace().getRoot().normalize(); + reply->getTrace().normalize(); // resending breaks the trace, so retry until it has expected form - if (!reply->hasErrors() && reply->getTrace().getRoot().encode() == expect.encode()) { + if (!reply->hasErrors() && reply->getTrace().encode() == expect.encode()) { break; } } @@ -116,7 +116,7 @@ Test::Main() } EXPECT_TRUE(!reply->hasErrors()); - EXPECT_EQUAL(reply->getTrace().getRoot().encode(), expect.encode()); + EXPECT_EQUAL(reply->getTrace().encode(), expect.encode()); EXPECT_TRUE(system((ctl_script + " stop all").c_str()) == 0); TEST_DONE(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index d0ee6229428..00327dc0002 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -94,7 +94,7 @@ public final class Node implements Nodelike { requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); if (parentHostname.isPresent()) { - if (!ipConfig.pool().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); + if (!ipConfig.pool().getIpSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 05bdfd25b76..86795767710 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -165,7 +165,7 @@ public class NodeRepository extends AbstractComponent { this.osVersions = new OsVersions(this); this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); - this.containerImages = new ContainerImages(db, containerImage, flagSource); + this.containerImages = new ContainerImages(db, containerImage); this.jobControl = new JobControl(new JobControlFlags(db, flagSource)); this.applications = new Applications(db); this.spareCount = spareCount; @@ -460,7 +460,7 @@ public class NodeRepository extends AbstractComponent { .map(node -> { if (node.state() != State.provisioned && node.state() != State.dirty) illegal("Can not set " + node + " ready. It is not provisioned or dirty."); - if (node.type() == NodeType.host && node.ipConfig().pool().isEmpty()) + if (node.type() == NodeType.host && node.ipConfig().pool().getIpSet().isEmpty()) illegal("Can not set host " + node + " ready. Its IP address pool is empty."); return node.withWantToRetire(false, false, Agent.system, clock.instant()); }) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index fd92b5b0ca0..847b825a7a4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -57,7 +57,7 @@ public class Application { public Application withCluster(ClusterSpec.Id id, boolean exclusive, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of()); + cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of(), ""); else cluster = cluster.withConfiguration(exclusive, min, max); return with(cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index a17ee081447..90133f7499e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -25,6 +25,7 @@ public class Cluster { private final Optional<ClusterResources> suggested; private final Optional<ClusterResources> target; private final List<ScalingEvent> scalingEvents; + private final String autoscalingStatus; public Cluster(ClusterSpec.Id id, boolean exclusive, @@ -32,7 +33,8 @@ public class Cluster { ClusterResources maxResources, Optional<ClusterResources> suggestedResources, Optional<ClusterResources> targetResources, - List<ScalingEvent> scalingEvents) { + List<ScalingEvent> scalingEvents, + String autoscalingStatus) { this.id = Objects.requireNonNull(id); this.exclusive = exclusive; this.min = Objects.requireNonNull(minResources); @@ -44,6 +46,7 @@ public class Cluster { else this.target = targetResources; this.scalingEvents = scalingEvents; + this.autoscalingStatus = autoscalingStatus; } public ClusterSpec.Id id() { return id; } @@ -73,21 +76,33 @@ public class Cluster { /** Returns the recent scaling events in this cluster */ public List<ScalingEvent> scalingEvents() { return scalingEvents; } + public Optional<ScalingEvent> lastScalingEvent() { + if (scalingEvents.isEmpty()) return Optional.empty(); + return Optional.of(scalingEvents.get(scalingEvents.size() - 1)); + } + + /** The latest autoscaling status of this cluster, or empty (never null) if none */ + public String autoscalingStatus() { return autoscalingStatus; } + public Cluster withConfiguration(boolean exclusive, ClusterResources min, ClusterResources max) { - return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } public Cluster withSuggested(Optional<ClusterResources> suggested) { - return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } public Cluster withTarget(Optional<ClusterResources> target) { - return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } public Cluster with(ScalingEvent scalingEvent) { // NOTE: We're just storing the latest scaling event so far - return new Cluster(id, exclusive, min, max, suggested, target, List.of(scalingEvent)); + return new Cluster(id, exclusive, min, max, suggested, target, List.of(scalingEvent), autoscalingStatus); + } + + public Cluster withAutoscalingStatus(String autoscalingStatus) { + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 1a8c4c8a6c2..d2c943794fe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -3,14 +3,16 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Duration; +import java.time.Instant; import java.util.List; +import java.util.Objects; import java.util.Optional; -import java.util.logging.Logger; /** * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster @@ -20,8 +22,6 @@ import java.util.logging.Logger; */ public class Autoscaler { - private static final Logger log = Logger.getLogger(Autoscaler.class.getName()); - /** What cost difference factor is worth a reallocation? */ private static final double costDifferenceWorthReallocation = 0.1; /** What difference factor for a resource is worth a reallocation? */ @@ -55,39 +55,46 @@ public class Autoscaler { * @return scaling advice for this cluster */ public Advice autoscale(Cluster cluster, List<Node> clusterNodes) { - if (cluster.minResources().equals(cluster.maxResources())) return Advice.none(); // Shortcut + if (cluster.minResources().equals(cluster.maxResources())) return Advice.none("Autoscaling is disabled"); // Shortcut return autoscale(cluster, clusterNodes, Limits.of(cluster), cluster.exclusive()); } private Advice autoscale(Cluster cluster, List<Node> clusterNodes, Limits limits, boolean exclusive) { - log.fine(() -> "Autoscale " + cluster.toString()); - - if (unstable(clusterNodes, nodeRepository)) { - log.fine(() -> "Unstable - Advice.none " + cluster.toString()); - return Advice.none(); - } + ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); + if (unstable(clusterNodes, nodeRepository)) + return Advice.none("Cluster change in progress"); - AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository, cluster.exclusive()); + AllocatableClusterResources currentAllocation = + new AllocatableClusterResources(clusterNodes, nodeRepository, cluster.exclusive()); ClusterTimeseries clusterTimeseries = new ClusterTimeseries(cluster, clusterNodes, metricsDb, nodeRepository); - Optional<Double> cpuLoad = clusterTimeseries.averageLoad(Resource.cpu, cluster); - Optional<Double> memoryLoad = clusterTimeseries.averageLoad(Resource.memory, cluster); - Optional<Double> diskLoad = clusterTimeseries.averageLoad(Resource.disk, cluster); - if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Advice.none(); + int measurementsPerNode = clusterTimeseries.measurementsPerNode(); + if (measurementsPerNode < minimumMeasurementsPerNode(clusterType)) + return Advice.none("Collecting more data before making new scaling decisions" + + ": Has " + measurementsPerNode + " data points per node"); - var target = ResourceTarget.idealLoad(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation); + int nodesMeasured = clusterTimeseries.nodesMeasured(); + if (nodesMeasured != clusterNodes.size()) + return Advice.none("Collecting more data before making new scaling decisions" + + ": Has measurements from " + nodesMeasured + " but need from " + clusterNodes.size()); + + double cpuLoad = clusterTimeseries.averageLoad(Resource.cpu); + double memoryLoad = clusterTimeseries.averageLoad(Resource.memory); + double diskLoad = clusterTimeseries.averageLoad(Resource.disk); + + var target = ResourceTarget.idealLoad(cpuLoad, memoryLoad, diskLoad, currentAllocation); Optional<AllocatableClusterResources> bestAllocation = allocationOptimizer.findBestAllocation(target, currentAllocation, limits, exclusive); - if (bestAllocation.isEmpty()) { - log.fine(() -> "bestAllocation.isEmpty: Advice.dontScale for " + cluster.toString()); - return Advice.dontScale(); - } - if (similar(bestAllocation.get(), currentAllocation)) { - log.fine(() -> "Current allocation similar: Advice.dontScale for " + cluster.toString()); - return Advice.dontScale(); - } + if (bestAllocation.isEmpty()) + return Advice.dontScale("No allocation changes are possible within configured limits"); + + if (similar(bestAllocation.get(), currentAllocation)) + return Advice.dontScale("Cluster is ideally scaled (within configured limits)"); + if (isDownscaling(bestAllocation.get(), currentAllocation) && recentlyScaled(cluster, clusterNodes)) + return Advice.dontScale("Waiting a while before scaling down"); + return Advice.scaleTo(bestAllocation.get().toAdvertisedClusterResources()); } @@ -106,10 +113,23 @@ public class Autoscaler { return Math.abs(r1 - r2) / (( r1 + r2) / 2) < threshold; } + /** Returns true if this reduces total resources in any dimension */ + private boolean isDownscaling(AllocatableClusterResources target, AllocatableClusterResources current) { + NodeResources targetTotal = target.toAdvertisedClusterResources().totalResources(); + NodeResources currentTotal = current.toAdvertisedClusterResources().totalResources(); + return ! targetTotal.justNumbers().satisfies(currentTotal.justNumbers()); + } + + private boolean recentlyScaled(Cluster cluster, List<Node> clusterNodes) { + Duration downscalingDelay = downscalingDelay(clusterNodes.get(0).allocation().get().membership().cluster().type()); + return cluster.lastScalingEvent().map(event -> event.at()).orElse(Instant.MIN) + .isAfter(nodeRepository.clock().instant().minus(downscalingDelay)); + } + /** The duration of the window we need to consider to make a scaling decision. See also minimumMeasurementsPerNode */ static Duration scalingWindow(ClusterSpec.Type clusterType) { if (clusterType.isContent()) return Duration.ofHours(12); - return Duration.ofHours(1); + return Duration.ofMinutes(30); } static Duration maxScalingWindow() { @@ -119,7 +139,16 @@ public class Autoscaler { /** Measurements are currently taken once a minute. See also scalingWindow */ static int minimumMeasurementsPerNode(ClusterSpec.Type clusterType) { if (clusterType.isContent()) return 60; - return 20; + return 7; + } + + /** + * We should wait a while before scaling down after a scaling event as a peak in usage + * indicates more peaks may arrive in the near future. + */ + static Duration downscalingDelay(ClusterSpec.Type clusterType) { + if (clusterType.isContent()) return Duration.ofHours(12); + return Duration.ofHours(1); } public static boolean unstable(List<Node> nodes, NodeRepository nodeRepository) { @@ -140,10 +169,12 @@ public class Autoscaler { private final boolean present; private final Optional<ClusterResources> target; + private final String reason; - private Advice(Optional<ClusterResources> target, boolean present) { + private Advice(Optional<ClusterResources> target, boolean present, String reason) { this.target = target; this.present = present; + this.reason = Objects.requireNonNull(reason); } /** @@ -158,10 +189,14 @@ public class Autoscaler { /** True if this provides advice (which may be to keep the current allocation) */ public boolean isPresent() { return present; } - private static Advice none() { return new Advice(Optional.empty(), false); } - private static Advice dontScale() { return new Advice(Optional.empty(), true); } - private static Advice scaleTo(ClusterResources target) { return new Advice(Optional.of(target), true); } + /** The reason for this advice */ + public String reason() { return reason; } + private static Advice none(String reason) { return new Advice(Optional.empty(), false, reason); } + private static Advice dontScale(String reason) { return new Advice(Optional.empty(), true, reason); } + private static Advice scaleTo(ClusterResources target) { + return new Advice(Optional.of(target), true, "Scaling due to load changes"); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java index bb91b77dce5..e325e797ca5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java @@ -10,8 +10,6 @@ import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -21,10 +19,7 @@ import java.util.stream.Collectors; */ public class ClusterTimeseries { - private static final Logger log = Logger.getLogger(ClusterTimeseries.class.getName()); - private final List<Node> clusterNodes; - private final Map<String, Instant> startTimePerNode; /** The measurements for all hosts in this snapshot */ private final List<NodeTimeseries> nodeTimeseries; @@ -32,9 +27,10 @@ public class ClusterTimeseries { public ClusterTimeseries(Cluster cluster, List<Node> clusterNodes, MetricsDb db, NodeRepository nodeRepository) { this.clusterNodes = clusterNodes; ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - this.nodeTimeseries = db.getNodeTimeseries(nodeRepository.clock().instant().minus(Autoscaler.scalingWindow(clusterType)), - clusterNodes.stream().map(Node::hostname).collect(Collectors.toSet())); - this.startTimePerNode = metricStartTimes(cluster, clusterNodes, nodeRepository); + var allTimeseries = db.getNodeTimeseries(nodeRepository.clock().instant().minus(Autoscaler.scalingWindow(clusterType)), + clusterNodes.stream().map(Node::hostname).collect(Collectors.toSet())); + Map<String, Instant> startTimePerNode = metricStartTimes(cluster, clusterNodes, allTimeseries, nodeRepository); + nodeTimeseries = filterStale(allTimeseries, startTimePerNode); } /** @@ -43,6 +39,7 @@ public class ClusterTimeseries { */ private Map<String, Instant> metricStartTimes(Cluster cluster, List<Node> clusterNodes, + List<NodeTimeseries> nodeTimeseries, NodeRepository nodeRepository) { Map<String, Instant> startTimePerHost = new HashMap<>(); if ( ! cluster.scalingEvents().isEmpty()) { @@ -65,31 +62,22 @@ public class ClusterTimeseries { return startTimePerHost; } - /** - * Returns the average load of this resource in the measurement window, - * or empty if we do not have a reliable measurement across the cluster nodes. - */ - public Optional<Double> averageLoad(Resource resource, Cluster cluster) { - ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - - List<NodeTimeseries> currentMeasurements = filterStale(nodeTimeseries, startTimePerNode); + /** Returns the average number of measurements per node */ + public int measurementsPerNode() { + int measurementCount = nodeTimeseries.stream().mapToInt(m -> m.size()).sum(); + return measurementCount / clusterNodes.size(); + } - // Require a total number of measurements scaling with the number of nodes, - // but don't require that we have at least that many from every node - int measurementCount = currentMeasurements.stream().mapToInt(m -> m.size()).sum(); - if (measurementCount / clusterNodes.size() < Autoscaler.minimumMeasurementsPerNode(clusterType)) { - log.fine(() -> "Too few measurements per node for " + cluster.toString() + ": measurementCount " + measurementCount + - " (" + nodeTimeseries.stream().mapToInt(m -> m.size()).sum() + " before filtering"); - return Optional.empty(); - } - if (currentMeasurements.size() != clusterNodes.size()) { - log.fine(() -> "Mssing measurements from some nodes for " + cluster.toString() + ": Has from " + currentMeasurements.size() + - "but need " + clusterNodes.size() + "(before filtering: " + nodeTimeseries.size() + ")"); - return Optional.empty(); - } + /** Returns the number of nodes measured in this */ + public int nodesMeasured() { + return nodeTimeseries.size(); + } - double measurementSum = currentMeasurements.stream().flatMap(m -> m.asList().stream()).mapToDouble(m -> value(resource, m)).sum(); - return Optional.of(measurementSum / measurementCount); + /** Returns the average load of this resource in this */ + public double averageLoad(Resource resource) { + int measurementCount = nodeTimeseries.stream().mapToInt(m -> m.size()).sum(); + double measurementSum = nodeTimeseries.stream().flatMap(m -> m.asList().stream()).mapToDouble(m -> value(resource, m)).sum(); + return measurementSum / measurementCount; } private double value(Resource resource, MetricSnapshot snapshot) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index b53f56e4743..809c54146d0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import java.time.Duration; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -70,13 +69,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Optional<Cluster> cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; - log.fine(() -> "Autoscale " + application.toString()); - var advice = autoscaler.autoscale(cluster.get(), clusterNodes); - if (advice.isEmpty()) return; - - if ( ! cluster.get().targetResources().equals(advice.target())) { + application = application.with(cluster.get().withAutoscalingStatus(advice.reason())); + if (advice.isEmpty()) { + applications().put(application, deployment.applicationLock().get()); + } + else if ( ! cluster.get().targetResources().equals(advice.target())) { applications().put(application.with(cluster.get().withTarget(advice.target())), deployment.applicationLock().get()); if (advice.target().isPresent()) { logAutoscaling(advice.target().get(), applicationId, cluster.get(), clusterNodes); @@ -100,11 +99,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { } static String toString(ClusterResources r) { - return String.format(Locale.US, "%d%s * [vcpu: %.1f, memory: %.1f Gb, disk %.1f Gb]" + - " (total: [vcpu: %.1f, memory: %.1f Gb, disk: %.1f Gb])", - r.nodes(), r.groups() > 1 ? " (in " + r.groups() + " groups)" : "", - r.nodeResources().vcpu(), r.nodeResources().memoryGb(), r.nodeResources().diskGb(), - r.nodes() * r.nodeResources().vcpu(), r.nodes() * r.nodeResources().memoryGb(), r.nodes() * r.nodeResources().diskGb()); + return r + " (total: " + r.totalResources() + ")"; } private Map<ClusterSpec.Id, List<Node>> nodesByCluster(List<Node> applicationNodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 3bf287a3e80..d2dcaaeae5b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; @@ -13,6 +14,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.curator.stats.LatencyMetrics; import com.yahoo.vespa.curator.stats.LockStats; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -26,12 +28,13 @@ import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; -import static com.yahoo.vespa.hosted.provision.Node.State.active; /** * @author oyving @@ -70,9 +73,39 @@ public class MetricsReporter extends NodeRepositoryMaintainer { updateLockMetrics(); updateDockerMetrics(nodes); updateTenantUsageMetrics(nodes); + updateRepairTicketMetrics(nodes); + updateAllocationMetrics(nodes); return true; } + private void updateAllocationMetrics(NodeList nodes) { + Map<ClusterKey, List<Node>> byCluster = nodes.stream() + .filter(node -> node.allocation().isPresent()) + .collect(Collectors.groupingBy(node -> new ClusterKey(node.allocation().get().owner(), node.allocation().get().membership().cluster().id()))); + byCluster.forEach((clusterKey, allocatedNodes) -> { + int activeNodes = 0; + int nonActiveNodes = 0; + for (var node : allocatedNodes) { + if (node.state() == State.active) { + activeNodes++; + } else { + nonActiveNodes++; + } + } + double nonActiveFraction; + if (activeNodes == 0) { // Cluster has been removed + nonActiveFraction = 1; + } else { + nonActiveFraction = (double) nonActiveNodes / (double) activeNodes; + } + Map<String, String> dimensions = new HashMap<>(dimensions(clusterKey.application)); + dimensions.put("clusterId", clusterKey.cluster.value()); + metric.set("nodes.active", activeNodes, getContext(dimensions)); + metric.set("nodes.nonActive", nonActiveNodes, getContext(dimensions)); + metric.set("nodes.nonActiveFraction", nonActiveFraction, getContext(dimensions)); + }); + } + private void updateZoneMetrics() { metric.set("zone.working", nodeRepository().isWorking() ? 1 : 0, null); } @@ -99,14 +132,12 @@ public class MetricsReporter extends NodeRepositoryMaintainer { Optional<Allocation> allocation = node.allocation(); if (allocation.isPresent()) { ApplicationId applicationId = allocation.get().owner(); - context = getContextAt( - "state", node.state().name(), - "host", node.hostname(), - "tenantName", applicationId.tenant().value(), - "applicationId", applicationId.serializedForm().replace(':', '.'), - "app", toApp(applicationId), - "clustertype", allocation.get().membership().cluster().type().name(), - "clusterid", allocation.get().membership().cluster().id().value()); + Map<String, String> dimensions = new HashMap<>(dimensions(applicationId)); + dimensions.put("state", node.state().name()); + dimensions.put("host", node.hostname()); + dimensions.put("clustertype", allocation.get().membership().cluster().type().name()); + dimensions.put("clusterid", allocation.get().membership().cluster().id().value()); + context = getContext(dimensions); long wantedRestartGeneration = allocation.get().restartGeneration().wanted(); metric.set("wantedRestartGeneration", wantedRestartGeneration, context); @@ -126,9 +157,8 @@ public class MetricsReporter extends NodeRepositoryMaintainer { currentVersion.get().equals(wantedVersion); metric.set("wantToChangeVespaVersion", converged ? 0 : 1, context); } else { - context = getContextAt( - "state", node.state().name(), - "host", node.hostname()); + context = getContext(Map.of("state", node.state().name(), + "host", node.hostname())); } Optional<Version> currentVersion = node.status().vespaVersion(); @@ -211,24 +241,16 @@ public class MetricsReporter extends NodeRepositoryMaintainer { return version.getMinor() + version.getMicro() / 1000.0; } - private Metric.Context getContextAt(String... point) { - if (point.length % 2 != 0) - throw new IllegalArgumentException("Dimension specification comes in pairs"); - - Map<String, String> dimensions = new HashMap<>(); - for (int i = 0; i < point.length; i += 2) { - dimensions.put(point[i], point[i + 1]); - } - + private Metric.Context getContext(Map<String, String> dimensions) { return contextMap.computeIfAbsent(dimensions, metric::createContext); } private void updateNodeCountMetrics(NodeList nodes) { - Map<Node.State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() - .collect(Collectors.groupingBy(Node::state)); + Map<State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() + .collect(Collectors.groupingBy(Node::state)); // Count per state - for (Node.State state : Node.State.values()) { + for (State state : State.values()) { List<Node> nodesInState = nodesByState.getOrDefault(state, List.of()); metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null); } @@ -237,7 +259,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { private void updateLockMetrics() { LockStats.getGlobal().getLockMetricsByPath() .forEach((lockPath, lockMetrics) -> { - Metric.Context context = getContextAt("lockPath", lockPath); + Metric.Context context = getContext(Map.of("lockPath", lockPath)); metric.set("lockAttempt.acquire", lockMetrics.getAndResetAcquireCount(), context); metric.set("lockAttempt.acquireFailed", lockMetrics.getAndResetAcquireFailedCount(), context); @@ -285,10 +307,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { .map(node -> node.allocation().get().requestedResources().justNumbers()) .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); - var context = getContextAt( - "tenantName", applicationId.tenant().value(), - "applicationId", applicationId.serializedForm().replace(':', '.'), - "app", toApp(applicationId)); + var context = getContext(dimensions(applicationId)); metric.set("hostedVespa.docker.allocatedCapacityCpu", allocatedCapacity.vcpu(), context); metric.set("hostedVespa.docker.allocatedCapacityMem", allocatedCapacity.memoryGb(), context); @@ -297,24 +316,65 @@ public class MetricsReporter extends NodeRepositoryMaintainer { ); } + private void updateRepairTicketMetrics(NodeList nodes) { + nodes.nodeType(NodeType.host).stream() + .map(node -> node.reports().getReport("repairTicket")) + .flatMap(Optional::stream) + .map(report -> report.getInspector().field("status").asString()) + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())) + .forEach((status, number) -> metric.set("hostedVespa.breakfixedHosts", number, getContext(Map.of("status", status)))); + } + + private static Map<String, String> dimensions(ApplicationId application) { + return Map.of("tenantName", application.tenant().value(), + "applicationId", application.serializedForm().replace(':', '.'), + "app", toApp(application)); + } + private static NodeResources getCapacityTotal(NodeList nodes) { - return nodes.hosts().state(active).asList().stream() - .map(host -> host.flavor().resources()) - .map(NodeResources::justNumbers) - .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); + return nodes.hosts().state(State.active).asList().stream() + .map(host -> host.flavor().resources()) + .map(NodeResources::justNumbers) + .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); } private static NodeResources getFreeCapacityTotal(NodeList nodes) { - return nodes.hosts().state(active).asList().stream() - .map(n -> freeCapacityOf(nodes, n)) - .map(NodeResources::justNumbers) - .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); + return nodes.hosts().state(State.active).asList().stream() + .map(n -> freeCapacityOf(nodes, n)) + .map(NodeResources::justNumbers) + .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add); } private static NodeResources freeCapacityOf(NodeList nodes, Node dockerHost) { return nodes.childrenOf(dockerHost).asList().stream() - .map(node -> node.flavor().resources().justNumbers()) - .reduce(dockerHost.flavor().resources().justNumbers(), NodeResources::subtract); + .map(node -> node.flavor().resources().justNumbers()) + .reduce(dockerHost.flavor().resources().justNumbers(), NodeResources::subtract); + } + + private static class ClusterKey { + + private final ApplicationId application; + private final ClusterSpec.Id cluster; + + public ClusterKey(ApplicationId application, ClusterSpec.Id cluster) { + this.application = application; + this.cluster = cluster; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterKey that = (ClusterKey) o; + return application.equals(that.application) && + cluster.equals(that.cluster); + } + + @Override + public int hashCode() { + return Objects.hash(application, cluster); + } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 41d6c1e5425..bac31c40418 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -20,6 +20,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.config.provision.NodeType.confighost; import static com.yahoo.config.provision.NodeType.controllerhost; @@ -254,18 +255,25 @@ public class IP { * @return an allocation from the pool, if any can be made */ public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver) { + if (ipAddresses.asSet().isEmpty()) { + // IP addresses have not yet been resolved and should be done later. + return findUnusedAddressStream(nodes) + .map(Allocation::ofAddress) + .findFirst(); + } + if (ipAddresses.protocol == IpAddresses.Protocol.ipv4) { - return findUnused(nodes).stream() + return findUnusedIpAddresses(nodes).stream() .findFirst() .map(addr -> Allocation.ofIpv4(addr, resolver)); } - var unusedAddresses = findUnused(nodes); + var unusedAddresses = findUnusedIpAddresses(nodes); var allocation = unusedAddresses.stream() .filter(IP::isV6) .findFirst() .map(addr -> Allocation.ofIpv6(addr, resolver)); - allocation.flatMap(Allocation::secondary).ifPresent(ipv4Address -> { + allocation.flatMap(Allocation::ipv4Address).ifPresent(ipv4Address -> { if (!unusedAddresses.contains(ipv4Address)) { throw new IllegalArgumentException("Allocation resolved " + ipv4Address + " from hostname " + allocation.get().hostname + @@ -276,17 +284,43 @@ public class IP { } /** - * Finds all unused addresses in this pool + * Finds all unused IP addresses in this pool * * @param nodes a list of all nodes in the repository */ - public Set<String> findUnused(NodeList nodes) { + public Set<String> findUnusedIpAddresses(NodeList nodes) { var unusedAddresses = new LinkedHashSet<>(getIpSet()); nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> getIpSet().contains(ip))) .forEach(node -> unusedAddresses.removeAll(node.ipConfig().primary())); return Collections.unmodifiableSet(unusedAddresses); } + /** + * Returns the number of unused IP addresses in the pool, assuming any and all unaccounted for hostnames + * in the pool are resolved to exactly 1 IP address (or 2 with {@link IpAddresses.Protocol#dualStack}). + */ + public int eventuallyUnusedAddressCount(NodeList nodes) { + // The address pool is filled immediately upon provisioning in dynamically provisioned zones, + // and within short time the IP address pool is filled. For all other cases, the IP address + // pool is already filled. + // + // The count in this method relies on the size of the IP address pool if that's non-empty, + // otherwise fall back to the address/hostname pool. + + + Set<String> currentIpAddresses = this.ipAddresses.asSet(); + if (!currentIpAddresses.isEmpty()) { + return findUnusedIpAddresses(nodes).size(); + } + + return (int) findUnusedAddressStream(nodes).count(); + } + + private Stream<Address> findUnusedAddressStream(NodeList nodes) { + Set<String> hostnames = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); + return addresses.stream().filter(address -> !hostnames.contains(address.hostname())); + } + public IpAddresses.Protocol getProtocol() { return ipAddresses.protocol; } @@ -299,10 +333,6 @@ public class IP { return addresses; } - public boolean isEmpty() { - return getIpSet().isEmpty(); - } - public Pool withIpAddresses(Set<String> ipAddresses) { return Pool.of(ipAddresses, addresses); } @@ -326,22 +356,17 @@ public class IP { } - /** An IP address allocation from a pool */ + /** An address allocation from a pool */ public static class Allocation { private final String hostname; - private final String primary; - private final Optional<String> secondary; - - private Allocation(String hostname, String primary, Optional<String> secondary) { - Objects.requireNonNull(primary, "primary must be non-null"); - Objects.requireNonNull(secondary, "ipv4Address must be non-null"); - if (secondary.isPresent() && !isV4(secondary.get())) { // Secondary must be IPv4, if present - throw new IllegalArgumentException("Invalid IPv4 address '" + secondary + "'"); - } + private final Optional<String> ipv4Address; + private final Optional<String> ipv6Address; + + private Allocation(String hostname, Optional<String> ipv4Address, Optional<String> ipv6Address) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); - this.primary = primary; - this.secondary = secondary; + this.ipv4Address = Objects.requireNonNull(ipv4Address, "ipv4Address must be non-null"); + this.ipv6Address = Objects.requireNonNull(ipv6Address, "ipv6Address must be non-null"); } /** @@ -350,13 +375,17 @@ public class IP { * A successful allocation is guaranteed to have an IPv6 address, but may also have an IPv4 address if the * hostname of the IPv6 address has an A record. * - * @param ipAddress Unassigned IPv6 address + * @param ipv6Address Unassigned IPv6 address * @param resolver DNS name resolver to use * @throws IllegalArgumentException if DNS is misconfigured * @return An allocation containing 1 IPv6 address and 1 IPv4 address (if hostname is dual-stack) */ - private static Allocation ofIpv6(String ipAddress, NameResolver resolver) { - String hostname6 = resolver.resolveHostname(ipAddress).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipAddress)); + private static Allocation ofIpv6(String ipv6Address, NameResolver resolver) { + if (!isV6(ipv6Address)) { + throw new IllegalArgumentException("Invalid IPv6 address '" + ipv6Address + "'"); + } + + String hostname6 = resolver.resolveHostname(ipv6Address).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipv6Address)); List<String> ipv4Addresses = resolver.resolveAll(hostname6).stream() .filter(IP::isV4) .collect(Collectors.toList()); @@ -369,10 +398,10 @@ public class IP { if (!hostname6.equals(hostname4)) { throw new IllegalArgumentException(String.format("Hostnames resolved from each IP address do not " + "point to the same hostname [%s -> %s, %s -> %s]", - ipAddress, hostname6, addr, hostname4)); + ipv6Address, hostname6, addr, hostname4)); } }); - return new Allocation(hostname6, ipAddress, ipv4Address); + return new Allocation(hostname6, ipv4Address, Optional.of(ipv6Address)); } /** @@ -391,7 +420,11 @@ public class IP { throw new IllegalArgumentException("Hostname " + hostname4 + " did not resolve to exactly 1 address. " + "Resolved: " + addresses); } - return new Allocation(hostname4, addresses.get(0), Optional.empty()); + return new Allocation(hostname4, Optional.of(addresses.get(0)), Optional.empty()); + } + + private static Allocation ofAddress(Address address) { + return new Allocation(address.hostname(), Optional.empty(), Optional.empty()); } /** Hostname pointing to the IP addresses in this */ @@ -399,27 +432,28 @@ public class IP { return hostname; } - /** Primary address of this allocation */ - public String primary() { - return primary; + /** IPv4 address of this allocation */ + public Optional<String> ipv4Address() { + return ipv4Address; } - /** Secondary address of this allocation */ - public Optional<String> secondary() { - return secondary; + /** IPv6 address of this allocation */ + public Optional<String> ipv6Address() { + return ipv6Address; } /** All IP addresses in this */ public Set<String> addresses() { ImmutableSet.Builder<String> builder = ImmutableSet.builder(); - secondary.ifPresent(builder::add); - builder.add(primary); + ipv4Address.ifPresent(builder::add); + ipv6Address.ifPresent(builder::add); return builder.build(); } @Override public String toString() { - return String.format("IP allocation [primary=%s, secondary=%s]", primary, secondary.orElse("<none>")); + return String.format("Address allocation [hostname=%s, IPv4=%s, IPv6=%s]", + hostname, ipv4Address.orElse("<none>"), ipv6Address.orElse("<none>")); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 2ddbd6def6f..3979b898145 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -47,6 +47,7 @@ public class ApplicationSerializer { private static final String groupsKey = "groups"; private static final String nodeResourcesKey = "resources"; private static final String scalingEventsKey = "scalingEvents"; + private static final String autoscalingStatusKey = "autoscalingStatus"; private static final String fromKey = "from"; private static final String toKey = "to"; private static final String generationKey = "generation"; @@ -95,6 +96,7 @@ public class ApplicationSerializer { cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey)); + clusterObject.setString(autoscalingStatusKey, cluster.autoscalingStatus()); } private static Cluster clusterFromSlime(String id, Inspector clusterObject) { @@ -104,7 +106,8 @@ public class ApplicationSerializer { clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey)), - scalingEventsFromSlime(clusterObject.field(scalingEventsKey))); + scalingEventsFromSlime(clusterObject.field(scalingEventsKey)), + clusterObject.field(autoscalingStatusKey).asString()); } private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java index 45156c57481..b4cb9158a5c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java @@ -6,9 +6,6 @@ import com.google.common.base.Suppliers; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Duration; @@ -31,7 +28,6 @@ public class ContainerImages { private final CuratorDatabaseClient db; private final DockerImage defaultImage; - private final BooleanFlag replaceImage; /** * The container image is read on every request to /nodes/v2/node/[fqdn]. Cache current images to avoid @@ -40,10 +36,9 @@ public class ContainerImages { */ private volatile Supplier<Map<NodeType, DockerImage>> images; - public ContainerImages(CuratorDatabaseClient db, DockerImage defaultImage, FlagSource flagSource) { + public ContainerImages(CuratorDatabaseClient db, DockerImage defaultImage) { this.db = db; this.defaultImage = defaultImage; - this.replaceImage = Flags.REGIONAL_CONTAINER_REGISTRY.bindTo(flagSource); createCache(); } @@ -85,7 +80,7 @@ public class ContainerImages { /** Rewrite the registry part of given image, using this zone's default image */ private DockerImage rewriteRegistry(DockerImage image) { DockerImage zoneImage = defaultImage; - if (zoneImage.replacedBy().isPresent() && replaceImage.value()) { + if (zoneImage.replacedBy().isPresent()) { zoneImage = zoneImage.replacedBy().get(); } return image.withRegistry(zoneImage.registry()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index b0baae650e4..6462fb6f19d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -71,47 +71,47 @@ public class GroupPreparer { } // There were some changes, so re-do the allocation with locks - try (Mutex lock = nodeRepository.lock(application)) { - try (Mutex allocationLock = nodeRepository.lockUnallocated()) { - NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - highestIndex, wantedGroups, allocationLock); - - if (nodeRepository.zone().getCloud().dynamicProvisioning()) { - Version osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); - List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit() - .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()), - deficit.getFlavor(), - application, - osVersion, - requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any)) - .orElseGet(List::of); - - // At this point we have started provisioning of the hosts, the first priority is to make sure that - // the returned hosts are added to the node-repo so that they are tracked by the provision maintainers - List<Node> hosts = provisionedHosts.stream() - .map(ProvisionedHost::generateHost) - .collect(Collectors.toList()); - nodeRepository.addNodes(hosts, Agent.application); - - // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit - List<NodeCandidate> candidates = provisionedHosts.stream() - .map(host -> NodeCandidate.createNewExclusiveChild(host.generateNode(), - host.generateHost())) - .collect(Collectors.toList()); - allocation.offer(candidates); - } - - if (! allocation.fulfilled() && requestedNodes.canFail()) - throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + - allocation.outOfCapacityDetails()); - - // Carry out and return allocation - nodeRepository.reserve(allocation.reservableNodes()); - nodeRepository.addDockerNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); - List<Node> acceptedNodes = allocation.finalNodes(); - surplusActiveNodes.removeAll(acceptedNodes); - return acceptedNodes; + try (Mutex lock = nodeRepository.lock(application); + Mutex allocationLock = nodeRepository.lockUnallocated()) { + + NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, + highestIndex, wantedGroups, allocationLock); + + if (nodeRepository.zone().getCloud().dynamicProvisioning()) { + Version osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); + List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit() + .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()), + deficit.getFlavor(), + application, + osVersion, + requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any)) + .orElseGet(List::of); + + // At this point we have started provisioning of the hosts, the first priority is to make sure that + // the returned hosts are added to the node-repo so that they are tracked by the provision maintainers + List<Node> hosts = provisionedHosts.stream() + .map(ProvisionedHost::generateHost) + .collect(Collectors.toList()); + nodeRepository.addNodes(hosts, Agent.application); + + // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit + List<NodeCandidate> candidates = provisionedHosts.stream() + .map(host -> NodeCandidate.createNewExclusiveChild(host.generateNode(), + host.generateHost())) + .collect(Collectors.toList()); + allocation.offer(candidates); } + + if (! allocation.fulfilled() && requestedNodes.canFail()) + throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + + allocation.outOfCapacityDetails()); + + // Carry out and return allocation + nodeRepository.reserve(allocation.reservableNodes()); + nodeRepository.addDockerNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); + List<Node> acceptedNodes = allocation.finalNodes(); + surplusActiveNodes.removeAll(acceptedNodes); + return acceptedNodes; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java index 96053fdaa91..af3bde02421 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java @@ -82,7 +82,11 @@ public class HostCapacity { * Number of free (not allocated) IP addresses assigned to the dockerhost. */ int freeIPs(Node dockerHost) { - return dockerHost.ipConfig().pool().findUnused(allNodes).size(); + if (dockerHost.type() == NodeType.host) { + return dockerHost.ipConfig().pool().eventuallyUnusedAddressCount(allNodes); + } else { + return dockerHost.ipConfig().pool().findUnusedIpAddresses(allNodes).size(); + } } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index f8231072a28..14937e6afeb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -363,11 +363,11 @@ abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidate> { try { allocation = parent.get().ipConfig().pool().findAllocation(allNodes, nodeRepository.nameResolver()); if (allocation.isEmpty()) return new InvalidNodeCandidate(resources, freeParentCapacity, parent.get(), - "No IP addresses available on parent host"); + "No addresses available on parent host"); } catch (Exception e) { - log.warning("Failed allocating IP address on " + parent.get() +": " + Exceptions.toMessageString(e)); + log.warning("Failed allocating address on " + parent.get() +": " + Exceptions.toMessageString(e)); return new InvalidNodeCandidate(resources, freeParentCapacity, parent.get(), - "Failed when allocating IP address on host"); + "Failed when allocating address on host"); } Node node = Node.createDockerNode(allocation.get().addresses(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 61cedbb9373..02621c79019 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -7,10 +7,12 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Status; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -26,25 +28,33 @@ public class ProvisionedHost { private final String hostHostname; private final Flavor hostFlavor; private final Optional<ApplicationId> exclusiveTo; - private final String nodeHostname; + private final List<Address> nodeAddresses; private final NodeResources nodeResources; private final Version osVersion; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, Optional<ApplicationId> exclusiveTo, - String nodeHostname, NodeResources nodeResources, Version osVersion) { + List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion) { this.id = Objects.requireNonNull(id, "Host id must be set"); this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.exclusiveTo = Objects.requireNonNull(exclusiveTo, "exclusiveTo must be set"); - this.nodeHostname = Objects.requireNonNull(nodeHostname, "Node hostname must be set"); + this.nodeAddresses = validateNodeAddresses(nodeAddresses); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); this.osVersion = Objects.requireNonNull(osVersion, "OS version must be set"); } + private static List<Address> validateNodeAddresses(List<Address> nodeAddresses) { + Objects.requireNonNull(nodeAddresses, "Node addresses must be set"); + if (nodeAddresses.isEmpty()) { + throw new IllegalArgumentException("There must be at least one node address"); + } + return nodeAddresses; + } + /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { Node.Builder builder = Node - .create(id, IP.Config.EMPTY, hostHostname, hostFlavor, NodeType.host) + .create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, NodeType.host) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))); exclusiveTo.ifPresent(builder::exclusiveTo); return builder.build(); @@ -52,7 +62,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the node running on this physical host */ public Node generateNode() { - return Node.createDockerNode(Set.of(), nodeHostname, hostHostname, nodeResources, NodeType.tenant).build(); + return Node.createDockerNode(Set.of(), nodeHostname(), hostHostname, nodeResources, NodeType.tenant).build(); } public String getId() { @@ -68,7 +78,11 @@ public class ProvisionedHost { } public String nodeHostname() { - return nodeHostname; + return nodeAddresses.get(0).hostname(); + } + + public List<Address> nodeAddresses() { + return nodeAddresses; } public NodeResources nodeResources() { return nodeResources; } @@ -81,14 +95,14 @@ public class ProvisionedHost { return id.equals(that.id) && hostHostname.equals(that.hostHostname) && hostFlavor.equals(that.hostFlavor) && - nodeHostname.equals(that.nodeHostname) && + nodeAddresses.equals(that.nodeAddresses) && nodeResources.equals(that.nodeResources) && osVersion.equals(that.osVersion); } @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, nodeHostname, nodeResources, osVersion); + return Objects.hash(id, hostHostname, hostFlavor, nodeAddresses, nodeResources, osVersion); } @Override @@ -97,7 +111,7 @@ public class ProvisionedHost { "id='" + id + '\'' + ", hostHostname='" + hostHostname + '\'' + ", hostFlavor=" + hostFlavor + - ", nodeHostname='" + nodeHostname + '\'' + + ", nodeAddresses='" + nodeAddresses + '\'' + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + '}'; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 9433b89ddc4..91b54fa37e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import java.net.URI; @@ -51,6 +52,8 @@ public class ApplicationSerializer { toSlime(currentResources, clusterObject.setObject("current")); cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); + scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); + clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); } private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { @@ -59,4 +62,13 @@ public class ApplicationSerializer { NodeResourcesSerializer.toSlime(resources.nodeResources(), clusterResourcesObject.setObject("resources")); } + private static void scalingEventsToSlime(List<ScalingEvent> scalingEvents, Cursor scalingEventsArray) { + for (ScalingEvent scalingEvent : scalingEvents) { + Cursor scalingEventObject = scalingEventsArray.addObject(); + toSlime(scalingEvent.from(), scalingEventObject.setObject("from")); + toSlime(scalingEvent.to(), scalingEventObject.setObject("to")); + scalingEventObject.setLong("at", scalingEvent.at().toEpochMilli()); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 304cebb3c01..c43629aeb09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -256,8 +257,12 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { Set<String> ipAddressPool = new HashSet<>(); inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); + List<Address> addressPool = new ArrayList<>(); + inspector.field("additionalHostnames").traverse((ArrayTraverser) (i, item) -> + addressPool.add(new Address(item.asString()))); + Node.Builder builder = Node.create(inspector.field("openStackId").asString(), - IP.Config.of(ipAddresses, ipAddressPool, List.of()), + IP.Config.of(ipAddresses, ipAddressPool, addressPool), inspector.field("hostname").asString(), flavorFromSlime(inspector), nodeTypeFromSlime(inspector.field("type"))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 5813a7067cd..5393aa7cfb8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import org.junit.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -44,11 +45,13 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 5, 1, hostResources); + tester.clock().advance(Duration.ofDays(1)); assertTrue("No measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.addCpuMeasurements(0.25f, 1f, 59, application1); assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.25f, 1f, 60, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", 15, 1, 1.3, 28.6, 28.6, @@ -58,6 +61,8 @@ public class AutoscalingTest { assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.deactivateRetired(application1, cluster1, scaledResources); + + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.8f, 1f, 3, application1); assertTrue("Load change is large, but insufficient measurements for new config -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); @@ -112,6 +117,7 @@ public class AutoscalingTest { tester.nodeRepository().getNodes(application1).stream() .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.slow); + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.25f, 1f, 120, application1); // Changing min and max from slow to any ClusterResources min = new ClusterResources( 2, 1, @@ -184,7 +190,7 @@ public class AutoscalingTest { } @Test - public void test_autoscaling_limits_when_min_equals_xax() { + public void test_autoscaling_limits_when_min_equals_max() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; @@ -195,6 +201,7 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 5, 1, resources); + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.25f, 1f, 120, application1); assertTrue(tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); } @@ -283,6 +290,31 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 6, 1, hostResources.withVcpu(hostResources.vcpu() / 2)); + tester.clock().advance(Duration.ofDays(1)); + tester.addMemMeasurements(0.02f, 0.95f, 120, application1); + tester.assertResources("Scaling down", + 6, 1, 2.8, 4.0, 95.0, + tester.autoscale(application1, cluster1.id(), min, max).target()); + } + + @Test + public void scaling_down_only_after_delay() { + NodeResources hostResources = new NodeResources(6, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(hostResources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); + + tester.deploy(application1, cluster1, 6, 1, hostResources.withVcpu(hostResources.vcpu() / 2)); + + // No autoscaling as it is too soon to scale down after initial deploy (counting as a scaling event) + tester.addMemMeasurements(0.02f, 0.95f, 120, application1); + assertTrue(tester.autoscale(application1, cluster1.id(), min, max).target().isEmpty()); + + // Trying the same a day later causes autoscaling + tester.clock().advance(Duration.ofDays(1)); tester.addMemMeasurements(0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", 6, 1, 2.8, 4.0, 95.0, @@ -344,6 +376,7 @@ public class AutoscalingTest { // deploy (Why 103 Gb memory? See AutoscalingTester.MockHostResourcesCalculator tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 103, 100, 1)); + tester.clock().advance(Duration.ofDays(1)); tester.addMemMeasurements(0.9f, 0.6f, 120, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", 8, 1, 3, 83, 34.3, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 4d8b6d13a86..3faa4c244ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; @@ -294,7 +295,7 @@ class AutoscalingTester { "hostname" + index, hostFlavor, Optional.empty(), - "nodename" + index, + List.of(new Address("nodename" + index)), resources, osVersion)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 5e318e00288..4b14174488e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -110,9 +110,8 @@ public class AutoscalingMaintainerTest { assertEquals(firstMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); // Add measurement of the expected generation, leading to rescaling - tester.clock().advance(Duration.ofSeconds(1)); + tester.clock().advance(Duration.ofHours(2)); tester.addMeasurements(0.1f, 0.1f, 0.1f, 1, 500, app1); - //tester.clock().advance(Duration.ofSeconds(1)); Instant lastMaintenanceTime = tester.clock().instant(); tester.maintainer().maintain(); assertEquals(lastMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); @@ -122,10 +121,10 @@ public class AutoscalingMaintainerTest { @Test public void test_toString() { - assertEquals("4 * [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb] (total: [vcpu: 4.0, memory: 8.0 Gb, disk: 16.0 Gb])", + assertEquals("4 nodes with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps])", AutoscalingMaintainer.toString(new ClusterResources(4, 1, new NodeResources(1, 2, 4, 1)))); - assertEquals("4 (in 2 groups) * [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb] (total: [vcpu: 4.0, memory: 8.0 Gb, disk: 16.0 Gb])", + assertEquals("4 nodes (in 2 groups) with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps])", AutoscalingMaintainer.toString(new ClusterResources(4, 2, new NodeResources(1, 2, 4, 1)))); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 478376bc0cd..2833c4e11ba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -208,12 +209,12 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); assertTrue("No IP addresses written as DNS updates are failing", - provisioning.get().stream().allMatch(host -> host.ipConfig().pool().isEmpty())); + provisioning.get().stream().allMatch(host -> host.ipConfig().pool().getIpSet().isEmpty())); tester.hostProvisioner.without(Behaviour.failDnsUpdate); tester.maintainer.maintain(); assertTrue("IP addresses written as DNS updates are succeeding", - provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().isEmpty())); + provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().getIpSet().isEmpty())); } private static class DynamicProvisioningTester { @@ -338,7 +339,7 @@ public class DynamicProvisioningMaintainerTest { "hostname" + index, hostFlavor, Optional.empty(), - "nodename" + index, + List.of(new Address("nodename" + index)), resources, osVersion)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index dbc0a98d879..a25858c034f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -3,14 +3,15 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; -import com.yahoo.test.ManualClock; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.applicationmodel.ApplicationInstance; @@ -46,6 +47,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; @@ -59,6 +61,8 @@ import static org.mockito.Mockito.when; */ public class MetricsReporterTest { + private static final Duration LONG_INTERVAL = Duration.ofDays(1); + private final ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); private final ApplicationInstanceReference reference = mock(ApplicationInstanceReference.class); @@ -138,7 +142,7 @@ public class MetricsReporterTest { orchestrator, serviceMonitor, () -> 42, - Duration.ofMinutes(1)); + LONG_INTERVAL); metricsReporter.maintain(); // Verify sum of values across dimensions, and remove these metrics to avoid checking against @@ -222,7 +226,7 @@ public class MetricsReporterTest { orchestrator, serviceMonitor, () -> 42, - Duration.ofMinutes(1)); + LONG_INTERVAL); metricsReporter.maintain(); assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts @@ -247,6 +251,53 @@ public class MetricsReporterTest { assertEquals(2.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityCpu", app2context), 0.01d); } + @Test + public void non_active_metric() { + ProvisioningTester tester = new ProvisioningTester.Builder().build(); + tester.makeReadyHosts(5, new NodeResources(64, 256, 2000, 10)); + tester.activateTenantHosts(); + TestMetric metric = new TestMetric(); + MetricsReporter metricsReporter = new MetricsReporter(tester.nodeRepository(), + metric, + tester.orchestrator(), + serviceMonitor, + () -> 42, + LONG_INTERVAL); + + + // Application is deployed + ApplicationId application = ApplicationId.from("t1", "a1", "default"); + Map<String, String> dimensions = Map.of("applicationId", application.toFullString()); + NodeResources resources = new NodeResources(2, 8, 100, 1); + List<Node> activeNodes = tester.deploy(application, Capacity.from(new ClusterResources(4, 1, resources))); + metricsReporter.maintain(); + assertEquals(0D, getMetric("nodes.nonActiveFraction", metric, dimensions)); + assertEquals(4, getMetric("nodes.active", metric, dimensions)); + assertEquals(0, getMetric("nodes.nonActive", metric, dimensions)); + + // One node fails + tester.fail(activeNodes.get(0).hostname()); + metricsReporter.maintain(); + assertEquals(0.33D, getMetric("nodes.nonActiveFraction", metric, dimensions).doubleValue(), 0.005); + assertEquals(3, getMetric("nodes.active", metric, dimensions)); + assertEquals(1, getMetric("nodes.nonActive", metric, dimensions)); + + // Cluster is removed + tester.deactivate(application); + metricsReporter.maintain(); + assertEquals(1D, getMetric("nodes.nonActiveFraction", metric, dimensions).doubleValue(), Double.MIN_VALUE); + assertEquals(0, getMetric("nodes.active", metric, dimensions)); + assertEquals(3, getMetric("nodes.nonActive", metric, dimensions)); + } + + private Number getMetric(String name, TestMetric metric, Map<String, String> dimensions) { + List<TestMetric.TestContext> metrics = metric.context.get(name).stream() + .filter(ctx -> ctx.properties.entrySet().containsAll(dimensions.entrySet())) + .collect(Collectors.toList()); + if (metrics.isEmpty()) throw new IllegalArgumentException("No value found for metric " + name + " with dimensions " + dimensions); + return metrics.get(metrics.size() - 1).value; + } + private ApplicationId app(String tenant) { return new ApplicationId.Builder() .tenant(tenant) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java index 09fb4d59443..b20524f678c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java @@ -10,8 +10,8 @@ import java.util.Map; public class TestMetric implements Metric { - public Map<String, Number> values = new LinkedHashMap<>(); - public Map<String, List<Context>> context = new LinkedHashMap<>(); + public final Map<String, Number> values = new LinkedHashMap<>(); + public final Map<String, List<TestContext>> context = new LinkedHashMap<>(); @Override public void set(String key, Number val, Context ctx) { @@ -74,9 +74,9 @@ public class TestMetric implements Metric { /** * Context where the propertymap is not shared - but unique to each value. */ - private static class TestContext implements Context{ + static class TestContext implements Context{ Number value; - Map<String, ?> properties; + final Map<String, ?> properties; public TestContext(Map<String, ?> properties) { this.properties = properties; @@ -86,4 +86,5 @@ public class TestMetric implements Metric { this.value = value; } } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index fb9c1ad0e5a..8101405ad7f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -86,8 +86,8 @@ public class IPTest { resolver.addReverseRecord("::2", "host1"); Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver); - assertEquals("::1", allocation.get().primary()); - assertFalse(allocation.get().secondary().isPresent()); + assertEquals(Optional.of("::1"), allocation.get().ipv6Address()); + assertFalse(allocation.get().ipv4Address().isPresent()); assertEquals("host3", allocation.get().hostname()); // Allocation fails if DNS record is missing @@ -105,16 +105,16 @@ public class IPTest { var pool = testPool(false); var allocation = pool.findAllocation(emptyList, resolver); assertFalse("Found allocation", allocation.isEmpty()); - assertEquals("127.0.0.1", allocation.get().primary()); - assertTrue("No secondary address", allocation.get().secondary().isEmpty()); + assertEquals(Optional.of("127.0.0.1"), allocation.get().ipv4Address()); + assertTrue("No IPv6 address", allocation.get().ipv6Address().isEmpty()); } @Test public void test_find_allocation_dual_stack() { IP.Pool pool = testPool(true); Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver); - assertEquals("::1", allocation.get().primary()); - assertEquals("127.0.0.2", allocation.get().secondary().get()); + assertEquals(Optional.of("::1"), allocation.get().ipv6Address()); + assertEquals("127.0.0.2", allocation.get().ipv4Address().get()); assertEquals("host3", allocation.get().hostname()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 72f9e9597de..e63f31cf304 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -33,7 +33,8 @@ public class ApplicationSerializerTest { new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), Optional.empty(), Optional.empty(), - List.of())); + List.of(), + "")); var minResources = new NodeResources(1, 2, 3, 4); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), true, @@ -44,7 +45,8 @@ public class ApplicationSerializerTest { List.of(new ScalingEvent(new ClusterResources(10, 5, minResources), new ClusterResources(12, 6, minResources), 7L, - Instant.ofEpochMilli(12345L))))); + Instant.ofEpochMilli(12345L))), + "Autoscaling status")); Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), clusters); @@ -65,6 +67,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.suggestedResources(), serializedCluster.suggestedResources()); assertEquals(originalCluster.targetResources(), serializedCluster.targetResources()); assertEquals(originalCluster.scalingEvents(), serializedCluster.scalingEvents()); + assertEquals(originalCluster.autoscalingStatus(), serializedCluster.autoscalingStatus()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java index d02244b7e11..9d390697df5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Test; @@ -52,22 +51,22 @@ public class ContainerImagesTest { @Test public void image_replacement() { var flagSource = new InMemoryFlagSource(); - var defaultImage = DockerImage.fromString("foo.example.com/vespa/vespa") - .withReplacedBy(DockerImage.fromString("bar.example.com/vespa/vespa")); + var defaultImage = DockerImage.fromString("foo.example.com/vespa/vespa"); var tester = new ProvisioningTester.Builder().defaultImage(defaultImage).flagSource(flagSource).build(); var hosts = tester.makeReadyNodes(2, "default", NodeType.host); tester.activateTenantHosts(); - // Default image is used with flag disabled - flagSource.withBooleanFlag(Flags.REGIONAL_CONTAINER_REGISTRY.id(), false); + // Default image is used when there is no replacement for (var host : hosts) { assertEquals(defaultImage, tester.nodeRepository().containerImages().imageFor(host.type())); } - // Enabling flag switches to replacement - flagSource.withBooleanFlag(Flags.REGIONAL_CONTAINER_REGISTRY.id(), true); + // Replacement image is preferred + DockerImage imageWithReplacement = defaultImage.withReplacedBy(DockerImage.fromString("bar.example.com/vespa/vespa")); + tester = new ProvisioningTester.Builder().defaultImage(imageWithReplacement).flagSource(flagSource).build(); + hosts = tester.makeReadyNodes(2, "default", NodeType.host); for (var host : hosts) { - assertEquals(defaultImage.replacedBy().get().asString(), + assertEquals(imageWithReplacement.replacedBy().get().asString(), tester.nodeRepository().containerImages().imageFor(host.type()).asString()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 4917a59879f..919d02c435c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -20,6 +20,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; @@ -471,7 +472,7 @@ public class DynamicDockerProvisionTest { throw new OutOfCapacityException("No host flavor matches " + resources); return provisionIndexes.stream() .map(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor.get(), Optional.empty(), - "host-" + i + "-1", resources, osVersion)) + List.of(new Address("host-" + i + "-1")), resources, osVersion)) .collect(Collectors.toList()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index c6e89680e85..808770f42dc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import org.junit.Before; import org.junit.Test; @@ -15,6 +16,8 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -32,8 +35,8 @@ public class HostCapacityTest { private HostCapacity capacity; private List<Node> nodes; private Node host1, host2, host3; - private final NodeResources resources1 = new NodeResources(1, 30, 20, 1.5); - private final NodeResources resources2 = new NodeResources(2, 40, 40, 0.5); + private final NodeResources dockerResources = new NodeResources(1, 30, 20, 1.5); + private final NodeResources docker2Resources = new NodeResources(2, 40, 40, 0.5); @Before public void setup() { @@ -48,15 +51,15 @@ public class HostCapacityTest { host3 = Node.create("host3", IP.Config.of(Set.of("::21"), generateIPs(22, 1), List.of()), "host3", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); // Add two containers to host1 - var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant).build(); - var nodeB = Node.createDockerNode(Set.of("::3"), "nodeB", "host1", resources1, NodeType.tenant).build(); + var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", dockerResources, NodeType.tenant).build(); + var nodeB = Node.createDockerNode(Set.of("::3"), "nodeB", "host1", dockerResources, NodeType.tenant).build(); // Add two containers to host 2 (same as host 1) - var nodeC = Node.createDockerNode(Set.of("::12"), "nodeC", "host2", resources1, NodeType.tenant).build(); - var nodeD = Node.createDockerNode(Set.of("::13"), "nodeD", "host2", resources1, NodeType.tenant).build(); + var nodeC = Node.createDockerNode(Set.of("::12"), "nodeC", "host2", dockerResources, NodeType.tenant).build(); + var nodeD = Node.createDockerNode(Set.of("::13"), "nodeD", "host2", dockerResources, NodeType.tenant).build(); // Add a larger container to host3 - var nodeE = Node.createDockerNode(Set.of("::22"), "nodeE", "host3", resources2, NodeType.tenant).build(); + var nodeE = Node.createDockerNode(Set.of("::22"), "nodeE", "host3", docker2Resources, NodeType.tenant).build(); // init docker host capacity nodes = new ArrayList<>(List.of(host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE)); @@ -65,19 +68,19 @@ public class HostCapacityTest { @Test public void hasCapacity() { - assertTrue(capacity.hasCapacity(host1, resources1)); - assertTrue(capacity.hasCapacity(host1, resources2)); - assertTrue(capacity.hasCapacity(host2, resources1)); - assertTrue(capacity.hasCapacity(host2, resources2)); - assertFalse(capacity.hasCapacity(host3, resources1)); // No ip available - assertFalse(capacity.hasCapacity(host3, resources2)); // No ip available + assertTrue(capacity.hasCapacity(host1, dockerResources)); + assertTrue(capacity.hasCapacity(host1, docker2Resources)); + assertTrue(capacity.hasCapacity(host2, dockerResources)); + assertTrue(capacity.hasCapacity(host2, docker2Resources)); + assertFalse(capacity.hasCapacity(host3, dockerResources)); // No ip available + assertFalse(capacity.hasCapacity(host3, docker2Resources)); // No ip available // Add a new node to host1 to deplete the memory resource - Node nodeF = Node.createDockerNode(Set.of("::6"), "nodeF", "host1", resources1, NodeType.tenant).build(); + Node nodeF = Node.createDockerNode(Set.of("::6"), "nodeF", "host1", dockerResources, NodeType.tenant).build(); nodes.add(nodeF); capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertFalse(capacity.hasCapacity(host1, resources1)); - assertFalse(capacity.hasCapacity(host1, resources2)); + assertFalse(capacity.hasCapacity(host1, dockerResources)); + assertFalse(capacity.hasCapacity(host1, docker2Resources)); } @Test @@ -112,19 +115,78 @@ public class HostCapacityTest { var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", nodeFlavors.getFlavorOrThrow("devhost"), NodeType.devhost).build(); - var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config).build(); + var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", dockerResources, NodeType.config).build(); var nodes = new ArrayList<>(List.of(cfg)); var capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertTrue(capacity.hasCapacity(devHost, resources1)); + assertTrue(capacity.hasCapacity(devHost, dockerResources)); - var container1 = Node.createDockerNode(Set.of("::3"), "container1", "devhost", resources1, NodeType.tenant).build(); + var container1 = Node.createDockerNode(Set.of("::3"), "container1", "devhost", dockerResources, NodeType.tenant).build(); nodes = new ArrayList<>(List.of(cfg, container1)); capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertFalse(capacity.hasCapacity(devHost, resources1)); + assertFalse(capacity.hasCapacity(devHost, dockerResources)); } + @Test + public void verifyCapacityFromAddresses() { + Node nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", dockerResources, NodeType.tenant).build(); + Node nodeB = Node.createDockerNode(Set.of("::3"), "nodeB", "host1", dockerResources, NodeType.tenant).build(); + Node nodeC = Node.createDockerNode(Set.of("::4"), "nodeC", "host1", dockerResources, NodeType.tenant).build(); + + // host1 is a host with resources = 7-100-120-5 (7 vcpus, 100G memory, 120G disk, and 5Gbps), + // while nodeA-C have resources = dockerResources = 1-30-20-1.5 + + Node host1 = setupHostWithAdditionalHostnames("host1", "nodeA"); + // Allocating nodeA should be OK + assertTrue(hasCapacity(dockerResources, host1)); + // then, the second node lacks hostname address + assertFalse(hasCapacity(dockerResources, host1, nodeA)); + + host1 = setupHostWithAdditionalHostnames("host1", "nodeA", "nodeB"); + // Allocating nodeA and nodeB should be OK + assertTrue(hasCapacity(dockerResources, host1)); + assertTrue(hasCapacity(dockerResources, host1, nodeA)); + // but the third node lacks hostname address + assertFalse(hasCapacity(dockerResources, host1, nodeA, nodeB)); + + host1 = setupHostWithAdditionalHostnames("host1", "nodeA", "nodeB", "nodeC"); + // Allocating nodeA, nodeB, and nodeC should be OK + assertTrue(hasCapacity(dockerResources, host1)); + assertTrue(hasCapacity(dockerResources, host1, nodeA)); + assertTrue(hasCapacity(dockerResources, host1, nodeA, nodeB)); + // but the fourth node lacks hostname address + assertFalse(hasCapacity(dockerResources, host1, nodeA, nodeB, nodeC)); + + host1 = setupHostWithAdditionalHostnames("host1", "nodeA", "nodeB", "nodeC", "nodeD"); + // Allocating nodeA, nodeB, and nodeC should be OK + assertTrue(hasCapacity(dockerResources, host1)); + assertTrue(hasCapacity(dockerResources, host1, nodeA)); + assertTrue(hasCapacity(dockerResources, host1, nodeA, nodeB)); + // but the fourth lacks memory (host has 100G, while 4x30G = 120G + assertFalse(hasCapacity(dockerResources, host1, nodeA, nodeB, nodeC)); + } + + private Node setupHostWithAdditionalHostnames(String hostHostname, String... additionalHostnames) { + List<Address> addresses = Stream.of(additionalHostnames).map(Address::new).collect(Collectors.toList()); + + doAnswer(invocation -> ((Flavor)invocation.getArguments()[0]).resources()) + .when(hostResourcesCalculator).advertisedResourcesOf(any()); + + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies( + "host", // 7-100-120-5 + "docker"); // 2- 40- 40-0.5 = docker2Resources + + return Node.create(hostHostname, IP.Config.of(Set.of("::1"), Set.of(), addresses), hostHostname, + nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); + } + + private boolean hasCapacity(NodeResources requestedCapacity, Node host, Node... remainingNodes) { + List<Node> nodes = Stream.concat(Stream.of(host), Stream.of(remainingNodes)).collect(Collectors.toList()); + var capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + return capacity.hasCapacity(host, requestedCapacity); + } + private Set<String> generateIPs(int start, int count) { // Allow 4 containers Set<String> ipAddressPool = new LinkedHashSet<>(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index f012f0a428f..b2529963a9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -314,8 +314,12 @@ public class ProvisioningTester { } public void fail(HostSpec host) { - int beforeFailCount = nodeRepository.getNode(host.hostname(), Node.State.active).get().status().failCount(); - Node failedNode = nodeRepository.fail(host.hostname(), Agent.system, "Failing to unit test"); + fail(host.hostname()); + } + + public void fail(String hostname) { + int beforeFailCount = nodeRepository.getNode(hostname, Node.State.active).get().status().failCount(); + Node failedNode = nodeRepository.fail(hostname, Agent.system, "Failing to unit test"); assertTrue(nodeRepository.getNodes(NodeType.tenant, Node.State.failed).contains(failedNode)); assertEquals(beforeFailCount + 1, failedNode.status().failCount()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index a98d383e219..86427fe30ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -91,8 +91,9 @@ public class NodesV2ApiTest { // POST new nodes assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address - asHostJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + - asNodeJson("parent2.yahoo.com", NodeType.host, "large-variant", Optional.of(TenantName.from("myTenant")), Optional.of(ApplicationId.from("tenant1", "app1", "instance1")), Optional.empty(), "127.0.127.1", "::127:1") + "," + + asHostJson("host9.yahoo.com", "large-variant", List.of("node9-1.yahoo.com"), "127.0.9.1", "::9:1") + "," + + asNodeJson("parent2.yahoo.com", NodeType.host, "large-variant", Optional.of(TenantName.from("myTenant")), + Optional.of(ApplicationId.from("tenant1", "app1", "instance1")), Optional.empty(), List.of(), "127.0.127.1", "::127:1") + "," + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), @@ -322,7 +323,7 @@ public class NodesV2ApiTest { // Attempt to POST host node with already assigned IP tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asHostJson("host200.yahoo.com", "default", "127.0.2.1") + "]", + "[" + asHostJson("host200.yahoo.com", "default", List.of(), "127.0.2.1") + "]", Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); @@ -334,7 +335,7 @@ public class NodesV2ApiTest { // Node types running a single container can share their IP address with child node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), "127.0.42.1") + "]", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), List.of(), "127.0.42.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -350,7 +351,7 @@ public class NodesV2ApiTest { // ... nor with child node on different host tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), "127.0.43.1") + "]", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), List.of(), "127.0.43.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", @@ -392,7 +393,7 @@ public class NodesV2ApiTest { @Test public void fails_to_ready_node_with_hard_fail() throws Exception { assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asHostJson("host12.yahoo.com", "default") + "]"). + ("[" + asHostJson("host12.yahoo.com", "default", List.of()) + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); @@ -961,7 +962,8 @@ public class NodesV2ApiTest { public void test_node_switch_hostname() throws Exception { String hostname = "host42.yahoo.com"; // Add host with switch hostname - String json = asNodeJson(hostname, NodeType.host, "default", Optional.empty(), Optional.empty(), Optional.of("switch0"), "127.0.42.1", "::42:1"); + String json = asNodeJson(hostname, NodeType.host, "default", Optional.empty(), Optional.empty(), + Optional.of("switch0"), List.of(), "127.0.42.1", "::42:1"); assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + json + "]").getBytes(StandardCharsets.UTF_8), Request.Method.POST), @@ -1013,17 +1015,22 @@ public class NodesV2ApiTest { "\"flavor\":\"" + flavor + "\"}"; } - private static String asHostJson(String hostname, String flavor, String... ipAddress) { - return asNodeJson(hostname, NodeType.host, flavor, Optional.empty(), Optional.empty(), Optional.empty(), ipAddress); + private static String asHostJson(String hostname, String flavor, List<String> additionalHostnames, String... ipAddress) { + return asNodeJson(hostname, NodeType.host, flavor, Optional.empty(), Optional.empty(), Optional.empty(), + additionalHostnames, ipAddress); } - private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, Optional<String> switchHostname, String... ipAddress) { + private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, + Optional<ApplicationId> exclusiveTo, Optional<String> switchHostname, + List<String> additionalHostnames, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + createIpAddresses(ipAddress) + "\"flavor\":\"" + flavor + "\"" + (reservedTo.map(tenantName -> ", \"reservedTo\":\"" + tenantName.value() + "\"").orElse("")) + (exclusiveTo.map(appId -> ", \"exclusiveTo\":\"" + appId.serializedForm() + "\"").orElse("")) + (switchHostname.map(s -> ", \"switchHostname\":\"" + s + "\"").orElse("")) + + (additionalHostnames.isEmpty() ? "" : ", \"additionalHostnames\":[\"" + + String.join("\",\"", additionalHostnames) + "\"]") + ", \"type\":\"" + nodeType + "\"}"; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json index 456ed18334e..82f7e04f92b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -62,7 +62,37 @@ "diskSpeed" : "fast", "storageType" : "any" } - } + }, + "scalingEvents" : [ + { + "from": { + "nodes": 0, + "groups": 0, + "resources": { + "vcpu" : 0.0, + "memoryGb": 0.0, + "diskGb": 0.0, + "bandwidthGbps": 0.0, + "diskSpeed": "fast", + "storageType": "any" + } + }, + "to": { + "nodes": 2, + "groups": 1, + "resources" : { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "local" + } + }, + "at" : 123 + } + ], + "autoscalingStatus" : "" } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json index bd22087ecfa..0ee590f60e0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json @@ -38,7 +38,37 @@ "diskSpeed": "fast", "storageType": "local" } - } + }, + "scalingEvents" : [ + { + "from": { + "nodes": 0, + "groups": 0, + "resources": { + "vcpu" : 0.0, + "memoryGb": 0.0, + "diskGb": 0.0, + "bandwidthGbps": 0.0, + "diskSpeed": "fast", + "storageType": "any" + } + }, + "to": { + "nodes": 2, + "groups": 1, + "resources" : { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "local" + } + }, + "at" : 123 + } + ], + "autoscalingStatus" : "" } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json index dac9fd30267..809e58bd7b6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json @@ -25,5 +25,6 @@ "127.0.9.1", "::9:1" ], - "additionalIpAddresses": [] + "additionalIpAddresses": [], + "additionalHostnames": ["node9-1.yahoo.com"] } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java index 4b82f278f23..e2e769f8556 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java @@ -14,6 +14,7 @@ import java.util.List; /** * @author bakksjo */ +@SuppressWarnings("removal") // VespaJerseyJaxRsClientFactory public class RetryingClusterControllerClientFactory extends AbstractComponent implements ClusterControllerClientFactory { // TODO: Figure this port out dynamically. diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java index 95fdd61563b..309d6a756f6 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactoryTest.java @@ -27,6 +27,7 @@ public class RetryingClusterControllerClientFactoryTest { private final Clock clock = new ManualClock(); @Test + @SuppressWarnings("removal") // VespaJerseyJaxRsClientFactory public void verifyJerseyCallForSetNodeState() throws IOException { VespaJerseyJaxRsClientFactory clientFactory = mock(VespaJerseyJaxRsClientFactory.class); ClusterControllerJaxRsApi api = mock(ClusterControllerJaxRsApi.class); diff --git a/parent/pom.xml b/parent/pom.xml index b3648ea52df..486f659e529 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -575,6 +575,11 @@ <version>${apache.httpcore.version}</version> </dependency> <dependency> + <groupId>org.apache.httpcomponents.client5</groupId> + <artifactId>httpclient5</artifactId> + <version>${apache.httpclient5.version}</version> + </dependency> + <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpmime</artifactId> <version>4.3.6</version> @@ -749,6 +754,7 @@ <antlr4.version>4.5</antlr4.version> <apache.httpclient.version>4.5.12</apache.httpclient.version> <apache.httpcore.version>4.4.13</apache.httpcore.version> + <apache.httpclient5.version>5.0.3</apache.httpclient5.version> <asm.version>7.0</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.version>1.8.49</athenz.version> diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 6ad74b660c1..96c55d0d9f2 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -31,8 +31,6 @@ using PersistenceProviderUP = std::unique_ptr<PersistenceProvider>; namespace { -LoadType defaultLoadType(0, "default"); - std::unique_ptr<PersistenceProvider> getSpi(ConformanceTest::PersistenceFactory &factory, const document::TestDocMan &testDocMan) { PersistenceProviderUP result(factory.getPersistenceImplementation( @@ -61,7 +59,7 @@ createIterator(PersistenceProvider& spi, fieldSet = std::make_shared<document::DocIdOnly>(); } - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); return spi.createIterator(b, std::move(fieldSet), sel, versions, context); } @@ -152,7 +150,7 @@ doIterate(PersistenceProvider& spi, std::vector<Chunk> chunks; while (true) { - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); IterateResult result(spi.iterate(id, maxByteSize, context)); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); @@ -204,7 +202,7 @@ iterateBucket(PersistenceProvider& spi, DocumentSelection docSel(""); Selection sel(docSel); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); CreateIteratorResult iter = spi.createIterator( bucket, std::make_shared<document::AllFields>(), @@ -286,7 +284,7 @@ feedDocs(PersistenceProvider& spi, uint32_t maxSize = 110) { std::vector<DocAndTimestamp> docs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < numDocs; ++i) { Document::SP doc( testDocMan.createRandomDocumentAtLocation( @@ -333,7 +331,7 @@ TEST_F(ConformanceTest, testBasics) _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); Document::SP doc2 = testDocMan.createRandomDocumentAtLocation(0x01, 2); @@ -428,7 +426,7 @@ TEST_F(ConformanceTest, testListBuckets) Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); Document::SP doc2 = testDocMan.createRandomDocumentAtLocation(0x02, 2); Document::SP doc3 = testDocMan.createRandomDocumentAtLocation(0x03, 3); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); spi->createBucket(bucket1, context); spi->createBucket(bucket2, context); spi->createBucket(bucket3, context); @@ -461,7 +459,7 @@ TEST_F(ConformanceTest, testBucketInfo) Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); Document::SP doc2 = testDocMan.createRandomDocumentAtLocation(0x01, 2); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); spi->createBucket(bucket, context); spi->put(bucket, Timestamp(2), doc2, context); @@ -518,7 +516,7 @@ TEST_F(ConformanceTest, testOrderIndependentBucketInfo) Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); Document::SP doc2 = testDocMan.createRandomDocumentAtLocation(0x01, 2); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); spi->createBucket(bucket, context); BucketChecksum checksumOrdered(0); @@ -557,7 +555,7 @@ TEST_F(ConformanceTest, testPut) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -582,7 +580,7 @@ TEST_F(ConformanceTest, testPutNewDocumentVersion) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -633,7 +631,7 @@ TEST_F(ConformanceTest, testPutOlderDocumentVersion) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -677,7 +675,7 @@ TEST_F(ConformanceTest, testPutDuplicate) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -709,7 +707,7 @@ TEST_F(ConformanceTest, testRemove) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -802,7 +800,7 @@ TEST_F(ConformanceTest, testRemoveMerge) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -897,7 +895,7 @@ TEST_F(ConformanceTest, testUpdate) _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); spi->createBucket(bucket, context); @@ -996,7 +994,7 @@ TEST_F(ConformanceTest, testGet) _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); spi->createBucket(bucket, context); @@ -1038,7 +1036,7 @@ TEST_F(ConformanceTest, testIterateCreateIterator) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1057,7 +1055,7 @@ TEST_F(ConformanceTest, testIterateWithUnknownId) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1071,7 +1069,7 @@ TEST_F(ConformanceTest, testIterateDestroyIterator) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1103,7 +1101,7 @@ TEST_F(ConformanceTest, testIterateAllDocs) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1121,7 +1119,7 @@ TEST_F(ConformanceTest, testIterateAllDocsNewestVersionOnly) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1149,7 +1147,7 @@ TEST_F(ConformanceTest, testIterateChunked) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1169,7 +1167,7 @@ TEST_F(ConformanceTest, testMaxByteSize) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1195,7 +1193,7 @@ TEST_F(ConformanceTest, testIterateMatchTimestampRange) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1233,7 +1231,7 @@ TEST_F(ConformanceTest, testIterateExplicitTimestampSubset) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1282,7 +1280,7 @@ TEST_F(ConformanceTest, testIterateRemoves) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1332,7 +1330,7 @@ TEST_F(ConformanceTest, testIterateMatchSelection) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1362,7 +1360,7 @@ TEST_F(ConformanceTest, testIterationRequiringDocumentIdOnlyMatching) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1392,7 +1390,7 @@ TEST_F(ConformanceTest, testIterateBadDocumentSelection) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); { @@ -1426,7 +1424,7 @@ TEST_F(ConformanceTest, testIterateAlreadyCompleted) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1450,7 +1448,7 @@ TEST_F(ConformanceTest, testIterateEmptyBucket) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); Selection sel(createSelection("")); @@ -1470,7 +1468,7 @@ TEST_F(ConformanceTest, testDeleteBucket) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -1495,7 +1493,7 @@ testDeleteBucketPostCondition(const PersistenceProvider &spi, const Bucket &bucket, const Document &doc1) { - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); { GetResult result = spi.get(bucket, document::AllFields(), @@ -1513,7 +1511,7 @@ TEST_F(ConformanceTest, testSplitNormalCase) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); Bucket bucketB(makeSpiBucket(BucketId(3, 0x06))); @@ -1555,7 +1553,7 @@ testSplitNormalCasePostCondition(const PersistenceProvider &spi, EXPECT_EQ(10, (int)spi.getBucketInfo(bucketB).getBucketInfo().getDocumentCount()); document::AllFields fs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); EXPECT_TRUE(spi.get(bucketA, fs, doc1->getId(), context).hasDocument()); @@ -1576,7 +1574,7 @@ TEST_F(ConformanceTest, testSplitTargetExists) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); Bucket bucketB(makeSpiBucket(BucketId(3, 0x06))); @@ -1633,7 +1631,7 @@ testSplitTargetExistsPostCondition(const PersistenceProvider &spi, getDocumentCount()); document::AllFields fs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); EXPECT_TRUE( @@ -1660,7 +1658,7 @@ TEST_F(ConformanceTest, testSplitSingleDocumentInSource) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket target1(makeSpiBucket(BucketId(3, 0x02))); Bucket target2(makeSpiBucket(BucketId(3, 0x06))); @@ -1696,7 +1694,7 @@ ConformanceTest::testSplitSingleDocumentInSourcePostCondition( EXPECT_EQ(uint32_t(1), spi.getBucketInfo(target2).getBucketInfo().getDocumentCount()); document::AllFields fs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Document::UP doc = testDocMan.createRandomDocumentAtLocation(0x06, 0); EXPECT_TRUE(spi.get(target2, fs, doc->getId(), context).hasDocument()); EXPECT_TRUE(!spi.get(target1, fs, doc->getId(), context).hasDocument()); @@ -1710,7 +1708,7 @@ ConformanceTest::createAndPopulateJoinSourceBuckets( const Bucket& source2, document::TestDocMan& testDocMan) { - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); spi.createBucket(source1, context); spi.createBucket(source2, context); @@ -1741,7 +1739,7 @@ ConformanceTest::doTestJoinNormalCase(const Bucket& source1, createAndPopulateJoinSourceBuckets(*spi, source1, source2, testDocMan); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); spi->join(source1, source2, target, context); testJoinNormalCasePostCondition(*spi, source1, source2, target, testDocMan); @@ -1780,7 +1778,7 @@ testJoinNormalCasePostCondition(const PersistenceProvider &spi, EXPECT_EQ(20, (int)spi.getBucketInfo(bucketC).getBucketInfo().getDocumentCount()); document::AllFields fs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc( testDocMan.createRandomDocumentAtLocation( @@ -1802,7 +1800,7 @@ TEST_F(ConformanceTest, testJoinTargetExists) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); spi->createBucket(bucketA, context); @@ -1851,7 +1849,7 @@ testJoinTargetExistsPostCondition(const PersistenceProvider &spi, EXPECT_EQ(30, (int)spi.getBucketInfo(bucketC).getBucketInfo().getDocumentCount()); document::AllFields fs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); EXPECT_TRUE(spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); @@ -1891,7 +1889,7 @@ TEST_F(ConformanceTest, testJoinOneBucket) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); spi->createBucket(bucketA, context); @@ -1921,7 +1919,7 @@ testJoinOneBucketPostCondition(const PersistenceProvider &spi, EXPECT_EQ(10, (int)spi.getBucketInfo(bucketC).getBucketInfo().getDocumentCount()); document::AllFields fs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); EXPECT_TRUE(spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); @@ -1947,7 +1945,7 @@ ConformanceTest::doTestJoinSameSourceBuckets(const Bucket& source, const Bucket& document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); spi->createBucket(source, context); populateBucket(source, *spi, context, 0, 10, testDocMan); @@ -1986,7 +1984,7 @@ ConformanceTest::testJoinSameSourceBucketsTargetExistsPostCondition( EXPECT_EQ(20, (int)spi.getBucketInfo(target).getBucketInfo().getDocumentCount()); document::AllFields fs; - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 20; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); EXPECT_TRUE(spi.get(target, fs, doc1->getId(), context).hasDocument()); @@ -1999,7 +1997,7 @@ TEST_F(ConformanceTest, testJoinSameSourceBucketsTargetExists) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket source(makeSpiBucket(BucketId(3, 0x02))); spi->createBucket(source, context); @@ -2038,7 +2036,7 @@ TEST_F(ConformanceTest, testBucketActivation) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); spi->setClusterState(makeBucketSpace(), createClusterState()); @@ -2081,7 +2079,7 @@ TEST_F(SingleDocTypeConformanceTest, testBucketActivationSplitAndJoin) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); Bucket bucketB(makeSpiBucket(BucketId(3, 0x06))); @@ -2159,7 +2157,7 @@ TEST_F(ConformanceTest, testRemoveEntry) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -2227,7 +2225,7 @@ TEST_F(ConformanceTest, testBucketSpaces) document::TestDocMan testDocMan; _factory->clear(); PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); + Context context(Priority(0), Trace::TraceLevel(0)); BucketSpace bucketSpace0(makeBucketSpace("testdoctype1")); BucketSpace bucketSpace1(makeBucketSpace("testdoctype2")); BucketSpace bucketSpace2(makeBucketSpace("no")); diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp index 429e2fb9d4e..80f5f05c59b 100644 --- a/persistence/src/vespa/persistence/spi/context.cpp +++ b/persistence/src/vespa/persistence/spi/context.cpp @@ -4,9 +4,8 @@ namespace storage::spi { -Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) - : _loadType(&loadType), - _priority(pri), +Context::Context(Priority pri, int maxTraceLevel) + : _priority(pri), _trace(maxTraceLevel), _readConsistency(ReadConsistency::STRONG) { } diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index 8c31439ee75..1fd7805b471 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -32,12 +32,8 @@ #include "read_consistency.h" #include <vespa/vespalib/trace/trace.h> -namespace metrics { class LoadType; } - namespace storage::spi { -using LoadType = metrics::LoadType; - typedef uint16_t Priority; // 0 - max pri, 255 - min pri // Define this type just because a ton of tests currently use it. @@ -46,18 +42,15 @@ struct Trace { }; class Context { - const LoadType* _loadType; Priority _priority; vespalib::Trace _trace; ReadConsistency _readConsistency; - public: Context(Context &&) = default; Context & operator = (Context &&) = default; - Context(const LoadType& loadType, Priority pri, int maxTraceLevel); + Context(Priority pri, int maxTraceLevel); ~Context(); - const LoadType& getLoadType() const { return *_loadType; } Priority getPriority() const { return _priority; } /** @@ -76,12 +69,14 @@ public: return _readConsistency; } + vespalib::Trace && steal_trace() { return std::move(_trace); } vespalib::Trace& getTrace() { return _trace; } const vespalib::Trace& getTrace() const { return _trace; } bool shouldTrace(int level) { return _trace.shouldTrace(level); } - void trace(int level, vespalib::stringref msg, bool addTime = true) - { _trace.trace(level, msg, addTime); } + void trace(int level, vespalib::stringref msg, bool addTime = true) { + _trace.trace(level, msg, addTime); + } }; } diff --git a/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.cpp index e6502d14347..714add169fd 100644 --- a/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.cpp +++ b/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.cpp @@ -23,7 +23,8 @@ DocumentApiMessageBusBmFeedHandler::DocumentApiMessageBusBmFeedHandler(BmMessage : IBmFeedHandler(), _name(vespalib::string("DocumentApiMessageBusBmFeedHandler(distributor)")), _storage_address(std::make_unique<StorageMessageAddress>("storage", NodeType::DISTRIBUTOR, 0)), - _message_bus(message_bus) + _message_bus(message_bus), + _route(_storage_address->to_mbus_route()) { } @@ -32,7 +33,7 @@ DocumentApiMessageBusBmFeedHandler::~DocumentApiMessageBusBmFeedHandler() = defa void DocumentApiMessageBusBmFeedHandler::send_msg(std::unique_ptr<documentapi::DocumentMessage> msg, PendingTracker& pending_tracker) { - _message_bus.send_msg(std::move(msg), _storage_address->getRoute(), pending_tracker); + _message_bus.send_msg(std::move(msg), _route, pending_tracker); } void diff --git a/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.h index 42bc61e587e..52e0b89007f 100644 --- a/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.h +++ b/searchcore/src/apps/vespa-feed-bm/document_api_message_bus_bm_feed_handler.h @@ -3,6 +3,7 @@ #pragma once #include "i_bm_feed_handler.h" +#include <vespa/messagebus/routing/route.h> namespace document { class DocumentTypeRepo; } namespace documentapi { class DocumentMessage; }; @@ -21,6 +22,7 @@ class DocumentApiMessageBusBmFeedHandler : public IBmFeedHandler vespalib::string _name; std::unique_ptr<storage::api::StorageMessageAddress> _storage_address; BmMessageBus& _message_bus; + mbus::Route _route; void send_msg(std::unique_ptr<documentapi::DocumentMessage> msg, PendingTracker& tracker); public: DocumentApiMessageBusBmFeedHandler(BmMessageBus &message_bus); diff --git a/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp index 24439a10925..daebc8a7a47 100644 --- a/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp +++ b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp @@ -6,7 +6,6 @@ #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/metrics/loadtype.h> #include <vespa/persistence/spi/persistenceprovider.h> using document::Document; @@ -20,8 +19,7 @@ namespace feedbm { namespace { -storage::spi::LoadType default_load_type(0, "default"); -storage::spi::Context context(default_load_type, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); +storage::spi::Context context(storage::spi::Priority(0), 0); void get_bucket_info_loop(PendingTracker &tracker) { @@ -38,7 +36,7 @@ class MyOperationComplete : public storage::spi::OperationComplete PendingTracker& _tracker; public: MyOperationComplete(std::atomic<uint32_t> &errors, const Bucket& bucket, PendingTracker& tracker); - ~MyOperationComplete(); + ~MyOperationComplete() override; void onComplete(std::unique_ptr<storage::spi::Result> result) override; void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; }; diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.cpp index 731b90888ea..18c1b979895 100644 --- a/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.cpp +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.cpp @@ -22,7 +22,8 @@ StorageApiMessageBusBmFeedHandler::StorageApiMessageBusBmFeedHandler(BmMessageBu _name(vespalib::string("StorageApiMessageBusBmFeedHandler(") + (distributor ? "distributor" : "service-layer") + ")"), _distributor(distributor), _storage_address(std::make_unique<StorageMessageAddress>("storage", distributor ? NodeType::DISTRIBUTOR : NodeType::STORAGE, 0)), - _message_bus(message_bus) + _message_bus(message_bus), + _route(_storage_address->to_mbus_route()) { } @@ -33,7 +34,7 @@ StorageApiMessageBusBmFeedHandler::send_msg(std::shared_ptr<storage::api::Storag { cmd->setSourceIndex(0); auto msg = std::make_unique<storage::mbusprot::StorageCommand>(cmd); - _message_bus.send_msg(std::move(msg), _storage_address->getRoute(), pending_tracker); + _message_bus.send_msg(std::move(msg), _route, pending_tracker); } void diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.h index 82447e1e873..6925053ad43 100644 --- a/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.h +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_message_bus_bm_feed_handler.h @@ -3,6 +3,7 @@ #pragma once #include "i_bm_feed_handler.h" +#include <vespa/messagebus/routing/route.h> namespace document { class DocumentTypeRepo; } namespace documentapi { class DocumentMessage; }; @@ -25,6 +26,7 @@ class StorageApiMessageBusBmFeedHandler : public IBmFeedHandler bool _distributor; std::unique_ptr<storage::api::StorageMessageAddress> _storage_address; BmMessageBus& _message_bus; + mbus::Route _route; void send_msg(std::shared_ptr<storage::api::StorageCommand> cmd, PendingTracker& tracker); public: StorageApiMessageBusBmFeedHandler(BmMessageBus &message_bus, bool distributor); diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index b875ab8e058..2b2b8acbc50 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -179,23 +179,20 @@ public: { } - void notifyPutDone(IDestructorCallbackSP, document::GlobalId gid, uint32_t lid, SerialNum) override { + void notifyPut(IDestructorCallbackSP, document::GlobalId gid, uint32_t lid, SerialNum) override { _changeGid = gid; _changeLid = lid; _gidToLid[gid] = lid; ++_changes; } - void notifyRemove(IDestructorCallbackSP, document::GlobalId gid, SerialNum) override { + void notifyRemove(IDestructorCallbackSP, document::GlobalId gid, SerialNum) override { _changeGid = gid; _changeLid = 0; _gidToLid[gid] = 0; ++_changes; } - void notifyRemoveDone(document::GlobalId, SerialNum) override { - } - void assertChanges(document::GlobalId expGid, uint32_t expLid, uint32_t expChanges) { EXPECT_EQUAL(expGid, _changeGid); EXPECT_EQUAL(expLid, _changeLid); diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 2b7bacfaec0..95a41a255ce 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -453,8 +453,7 @@ TEST_F("require that getPartitionStates() prepares all handlers", SimpleFixture) TEST_F("require that puts are routed to handler", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.put(bucket1, tstamp1, doc1, context); TEST_DO(assertHandler(bucket1, tstamp1, docId1, f.hset.handler1)); TEST_DO(assertHandler(bucket0, tstamp0, docId0, f.hset.handler2)); @@ -473,8 +472,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur f._writeFilter._acceptWriteOperation = false; f._writeFilter._message = "Disk is full"; - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::ErrorType::RESOURCE_EXHAUSTED, "Put operation rejected for document 'id:type3:type3::1': 'Disk is full'"), @@ -484,8 +482,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur TEST_F("require that updates are routed to handler", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.setExistingTimestamp(tstamp2); UpdateResult ur = f.engine.update(bucket1, tstamp1, upd1, context); TEST_DO(assertHandler(bucket1, tstamp1, docId1, f.hset.handler1)); @@ -504,8 +501,7 @@ TEST_F("require that updates are routed to handler", SimpleFixture) TEST_F("require that updates with bad ids are rejected", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL(UpdateResult(Result::ErrorType::PERMANENT_ERROR, "Update operation rejected due to bad id (id:type2:type2::1, type1)"), f.engine.update(bucket1, tstamp1, bad_id_upd, context)); @@ -516,8 +512,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix f._writeFilter._acceptWriteOperation = false; f._writeFilter._message = "Disk is full"; - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::ErrorType::RESOURCE_EXHAUSTED, @@ -528,8 +523,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix TEST_F("require that removes are routed to handlers", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); RemoveResult rr = f.engine.remove(bucket1, tstamp1, docId3, context); TEST_DO(assertHandler(bucket0, tstamp0, docId0, f.hset.handler1)); TEST_DO(assertHandler(bucket0, tstamp0, docId0, f.hset.handler2)); @@ -565,8 +559,7 @@ TEST_F("require that remove is NOT rejected if resource limit is reached", Simpl f._writeFilter._acceptWriteOperation = false; f._writeFilter._message = "Disk is full"; - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL(RemoveResult(false), f.engine.remove(bucket1, tstamp1, docId1, context)); } @@ -618,8 +611,7 @@ TEST_F("require that getBucketInfo() is routed to handlers and merged", SimpleFi TEST_F("require that createBucket() is routed to handlers and merged", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1._createBucketResult = Result(Result::ErrorType::TRANSIENT_ERROR, "err1a"); f.hset.handler2._createBucketResult = Result(Result::ErrorType::PERMANENT_ERROR, "err2a"); @@ -631,8 +623,7 @@ TEST_F("require that createBucket() is routed to handlers and merged", SimpleFix TEST_F("require that deleteBucket() is routed to handlers and merged", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.deleteBucketResult = Result(Result::ErrorType::TRANSIENT_ERROR, "err1"); f.hset.handler2.deleteBucketResult = Result(Result::ErrorType::PERMANENT_ERROR, "err2"); @@ -650,8 +641,7 @@ TEST_F("require that getModifiedBuckets() is routed to handlers and merged", Sim TEST_F("require that get is sent to all handlers", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); @@ -661,8 +651,7 @@ TEST_F("require that get is sent to all handlers", SimpleFixture) { TEST_F("require that get freezes the bucket", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.wasFrozen(bucket1)); - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_TRUE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_TRUE(f.hset.handler2.wasFrozen(bucket1)); @@ -673,8 +662,7 @@ TEST_F("require that get freezes the bucket", SimpleFixture) { TEST_F("require that get returns the first document found", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); f.hset.handler2.setDocument(*doc2, tstamp2); - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); @@ -687,8 +675,7 @@ TEST_F("require that get returns the first document found", SimpleFixture) { } TEST_F("require that createIterator does", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -701,8 +688,7 @@ TEST_F("require that createIterator does", SimpleFixture) { } TEST_F("require that iterator ids are unique", SimpleFixture) { - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -716,8 +702,7 @@ TEST_F("require that iterator ids are unique", SimpleFixture) { TEST_F("require that iterate requires valid iterator", SimpleFixture) { uint64_t max_size = 1024; - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, context); EXPECT_TRUE(it_result.hasError()); EXPECT_EQUAL(Result::ErrorType::PERMANENT_ERROR, it_result.getErrorCode()); @@ -736,8 +721,7 @@ TEST_F("require that iterate returns documents", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); f.hset.handler2.setDocument(*doc2, tstamp2); - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); uint64_t max_size = 1024; CreateIteratorResult result = f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, @@ -752,8 +736,7 @@ TEST_F("require that iterate returns documents", SimpleFixture) { TEST_F("require that destroyIterator prevents iteration", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -773,8 +756,7 @@ TEST_F("require that destroyIterator prevents iteration", SimpleFixture) { TEST_F("require that buckets are frozen during iterator life", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1)); - storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index a10d48ee7fe..9d72045c918 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/server/executor_thread_service.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_handler.h> #include <vespa/searchlib/common/gatecallback.h> #include <map> @@ -112,11 +113,13 @@ public: struct Fixture { std::vector<std::shared_ptr<ListenerStats>> _statss; - std::shared_ptr<GidToLidChangeHandler> _handler; + std::shared_ptr<GidToLidChangeHandler> _real_handler; + std::shared_ptr<IGidToLidChangeHandler> _handler; Fixture() : _statss(), - _handler(std::make_shared<GidToLidChangeHandler>()) + _real_handler(std::make_shared<GidToLidChangeHandler>()), + _handler(_real_handler) { } @@ -127,7 +130,7 @@ struct Fixture void close() { - _handler->close(); + _real_handler->close(); } ListenerStats &addStats() { @@ -139,10 +142,15 @@ struct Fixture _handler->addListener(std::move(listener)); } - void notifyPutDone(GlobalId gid, uint32_t lid, SerialNum serialNum) { - vespalib::Gate gate; - _handler->notifyPutDone(std::make_shared<search::GateCallback>(gate), gid, lid, serialNum); - gate.await(); + void commit() { + auto pending = _handler->grab_pending_changes(); + if (pending) { + pending->notify_done(); + } + } + + void notifyPut(GlobalId gid, uint32_t lid, SerialNum serial_num) { + _handler->notifyPut(std::shared_ptr<search::IDestructorCallback>(), gid, lid, serial_num); } void notifyRemove(GlobalId gid, SerialNum serialNum) { @@ -151,10 +159,6 @@ struct Fixture gate.await(); } - void notifyRemoveDone(GlobalId gid, SerialNum serialNum) { - _handler->notifyRemoveDone(gid, serialNum); - } - void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) { _handler->removeListeners(docTypeName, keepNames); @@ -169,7 +173,8 @@ TEST_F("Test that we can register a listener", Fixture) TEST_DO(stats.assertListeners(1, 0, 0)); f.addListener(std::move(listener)); TEST_DO(stats.assertListeners(1, 1, 0)); - f.notifyPutDone(toGid(doc1), 10, 10); + f.notifyPut(toGid(doc1), 10, 10); + f.commit(); TEST_DO(stats.assertChanges(1, 0)); f.removeListeners("testdoc", {}); TEST_DO(stats.assertListeners(1, 1, 1)); @@ -192,7 +197,8 @@ TEST_F("Test that we can register multiple listeners", Fixture) TEST_DO(stats1.assertListeners(1, 1, 0)); TEST_DO(stats2.assertListeners(1, 1, 0)); TEST_DO(stats3.assertListeners(1, 1, 0)); - f.notifyPutDone(toGid(doc1), 10, 10); + f.notifyPut(toGid(doc1), 10, 10); + f.commit(); TEST_DO(stats1.assertChanges(1, 0)); TEST_DO(stats2.assertChanges(1, 0)); TEST_DO(stats3.assertChanges(1, 0)); @@ -250,62 +256,39 @@ public: } }; -TEST_F("Test that put is ignored if we have a pending remove", StatsFixture) +TEST_F("Test that multiple puts are processed", StatsFixture) { - f.notifyRemove(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 10, 10); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 11, 30); - TEST_DO(f.assertChanges(1, 1)); + f.notifyPut(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 0)); + f.notifyPut(toGid(doc1), 11, 20); + TEST_DO(f.assertChanges(0, 0)); + f.commit(); + TEST_DO(f.assertChanges(2, 0)); } -TEST_F("Test that pending removes are merged", StatsFixture) +TEST_F("Test that put is ignored if we have a pending remove", StatsFixture) { + f.notifyPut(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 0)); f.notifyRemove(toGid(doc1), 20); TEST_DO(f.assertChanges(0, 1)); - f.notifyRemove(toGid(doc1), 40); + f.commit(); TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 10, 10); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 11, 30); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 40); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 12, 50); + f.notifyPut(toGid(doc1), 11, 30); + f.commit(); TEST_DO(f.assertChanges(1, 1)); } -TEST_F("Test that out of order notifyRemoveDone is handled", StatsFixture) +TEST_F("Test that pending removes are merged", StatsFixture) { - f.notifyRemove(toGid(doc1), 20); + f.notifyPut(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 0)); + f.notifyRemove(toGid(doc1), 20); TEST_DO(f.assertChanges(0, 1)); f.notifyRemove(toGid(doc1), 40); TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 40); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 20); + f.commit(); TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 12, 50); - TEST_DO(f.assertChanges(1, 1)); -} - -TEST_F("Test that out of order notifyPutDone is partially handled", StatsFixture) -{ - f.notifyRemove(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 12, 50); - TEST_DO(f.assertChanges(1, 1)); - f.notifyPutDone(toGid(doc1), 11, 40); - TEST_DO(f.assertChanges(1, 1)); - f.notifyPutDone(toGid(doc1), 13, 55); - TEST_DO(f.assertChanges(2, 1)); - f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(f.assertChanges(2, 1)); } } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 5ace4b66803..08bb68b1b40 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -499,4 +499,4 @@ bucketdb.checksumtype enum {LEGACY, XXHASH64} default = LEGACY restart ## TENSOR_ENGINE (default) uses DefaultTensorEngine, which has been the production implementation for years. ## FAST_VALUE uses the new and optimized FastValueBuilderFactory instead. ## TODO: Remove when default has been switched to FAST_VALUE. -tensor_implementation enum {TENSOR_ENGINE, FAST_VALUE} default = TENSOR_ENGINE +tensor_implementation enum {TENSOR_ENGINE, FAST_VALUE} default = FAST_VALUE diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 0eec650cfdb..02efd1ef11f 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -611,9 +611,7 @@ PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Buck void PersistenceEngine::destroyIterators() { - Context context(storage::spi::LoadType(0, "default"), - storage::spi::Priority(0x80), - storage::spi::Trace::TraceLevel(0)); + Context context(storage::spi::Priority(0x80), 0); for (;;) { IteratorId id; { diff --git a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt index a98b095cc21..1ba96e9adf5 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt @@ -10,7 +10,7 @@ vespa_add_library(searchcore_reference STATIC gid_to_lid_change_registrator.cpp gid_to_lid_mapper.cpp gid_to_lid_mapper_factory.cpp - pending_notify_remove_done.cpp + pending_gid_to_lid_changes.cpp DEPENDS searchcore_attribute searchcore_documentmetastore diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp index 6c45096a53f..a579a11e61f 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dummy_gid_to_lid_change_handler.h" +#include "i_pending_gid_to_lid_changes.h" namespace proton { @@ -9,7 +10,7 @@ DummyGidToLidChangeHandler::DummyGidToLidChangeHandler() = default; DummyGidToLidChangeHandler::~DummyGidToLidChangeHandler() = default; void -DummyGidToLidChangeHandler::notifyPutDone(IDestructorCallbackSP , GlobalId, uint32_t, SerialNum) +DummyGidToLidChangeHandler::notifyPut(IDestructorCallbackSP, GlobalId, uint32_t, SerialNum) { } @@ -18,9 +19,10 @@ DummyGidToLidChangeHandler::notifyRemove(IDestructorCallbackSP , GlobalId, Seria { } -void -DummyGidToLidChangeHandler::notifyRemoveDone(GlobalId, SerialNum) +std::unique_ptr<IPendingGidToLidChanges> +DummyGidToLidChangeHandler::grab_pending_changes() { + return {}; } void diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h index d5f6d788885..54cc0e2144a 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h @@ -22,11 +22,11 @@ public: DummyGidToLidChangeHandler(); ~DummyGidToLidChangeHandler() override; - void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) override; + void notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) override; void notifyRemove(IDestructorCallbackSP context, GlobalId gid, SerialNum serialNum) override; - void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override; void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) override; + std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp index 0c9087405b6..805f19b5bf0 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp @@ -2,6 +2,7 @@ #include "gid_to_lid_change_handler.h" #include "i_gid_to_lid_change_listener.h" +#include "pending_gid_to_lid_changes.h" #include <vespa/vespalib/util/lambdatask.h> #include <cassert> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -14,8 +15,8 @@ GidToLidChangeHandler::GidToLidChangeHandler() : _lock(), _listeners(), _closed(false), - _pendingRemove() - + _pendingRemove(), + _pending_changes() { } @@ -43,6 +44,13 @@ GidToLidChangeHandler::notifyRemove(IDestructorCallbackSP context, GlobalId gid) } void +GidToLidChangeHandler::notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) +{ + lock_guard guard(_lock); + _pending_changes.emplace_back(std::move(context), gid, lid, serial_num, false); +} + +void GidToLidChangeHandler::notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) { lock_guard guard(_lock); @@ -79,6 +87,7 @@ GidToLidChangeHandler::notifyRemove(IDestructorCallbackSP context, GlobalId gid, } else { notifyRemove(std::move(context), gid); } + _pending_changes.emplace_back(IDestructorCallbackSP(), gid, 0, serialNum, true); } void @@ -96,6 +105,16 @@ GidToLidChangeHandler::notifyRemoveDone(GlobalId gid, SerialNum serialNum) } } +std::unique_ptr<IPendingGidToLidChanges> +GidToLidChangeHandler::grab_pending_changes() +{ + lock_guard guard(_lock); + if (_pending_changes.empty()) { + return {}; + } + return std::make_unique<PendingGidToLidChanges>(*this, std::move(_pending_changes)); +} + void GidToLidChangeHandler::close() { diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h index 13a51edd0b5..25645aebcc9 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h @@ -3,8 +3,8 @@ #pragma once #include "i_gid_to_lid_change_handler.h" +#include "pending_gid_to_lid_change.h" #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/document/base/globalid.h> #include <vector> #include <mutex> @@ -44,6 +44,7 @@ class GidToLidChangeHandler : public std::enable_shared_from_this<GidToLidChange Listeners _listeners; bool _closed; vespalib::hash_map<GlobalId, PendingRemoveEntry, GlobalId::hash> _pendingRemove; + std::vector<PendingGidToLidChange> _pending_changes; void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid); void notifyRemove(IDestructorCallbackSP context, GlobalId gid); @@ -51,9 +52,11 @@ public: GidToLidChangeHandler(); ~GidToLidChangeHandler() override; - void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) override; + void notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) override; + void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum); void notifyRemove(IDestructorCallbackSP context, GlobalId gid, SerialNum serialNum) override; - void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; + void notifyRemoveDone(GlobalId gid, SerialNum serialNum); + std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() override; /** * Close handler, further notifications are blocked. diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h index cbafef57e46..0dad58b31ae 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h @@ -10,6 +10,8 @@ namespace document { class GlobalId; } namespace proton { +class IPendingGidToLidChanges; + /* * Interface class for registering listeners that get notification when * gid to lid mapping changes. @@ -36,11 +38,15 @@ public: virtual void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) = 0; /** - * Notify gid to lid mapping change. + * Notify pending gid to lid mapping change. Passed on to listeners later + * when force commit has made changes visible. + */ + virtual void notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) = 0; + /** + * Notify removal of gid. Passed on to listeners at once. */ - virtual void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) = 0; virtual void notifyRemove(IDestructorCallbackSP context, GlobalId gid, SerialNum serialNum) = 0; - virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) = 0; + virtual std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h b/searchcore/src/vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h new file mode 100644 index 00000000000..6e4e9f240ff --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace proton { + +/* + * Interface class for a container of gid to lid changes awaiting a + * force commit. + */ +class IPendingGidToLidChanges +{ +public: + virtual ~IPendingGidToLidChanges() = default; + virtual void notify_done() = 0; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_change.h b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_change.h new file mode 100644 index 00000000000..1577077a47b --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_change.h @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/document/base/globalid.h> +#include <vespa/searchlib/common/serialnum.h> + +namespace proton { + +/* + * Class for a gid to lid change awaiting a force commit. + */ +class PendingGidToLidChange +{ + using Context = std::shared_ptr<search::IDestructorCallback>; + using GlobalId = document::GlobalId; + using SerialNum = search::SerialNum; + + Context _context; + SerialNum _serial_num; + GlobalId _gid; + uint32_t _lid; + bool _is_remove; +public: + PendingGidToLidChange(); + PendingGidToLidChange(Context context, const GlobalId& gid, uint32_t lid, SerialNum serial_num, bool is_remove_) noexcept + : _context(std::move(context)), + _serial_num(serial_num), + _gid(gid), + _lid(lid), + _is_remove(is_remove_) + { + } + ~PendingGidToLidChange() = default; + + Context steal_context() && { return std::move(_context); } + const GlobalId &get_gid() const { return _gid; } + uint32_t get_lid() const { return _lid; } + SerialNum get_serial_num() const { return _serial_num; } + bool is_remove() const { return _is_remove; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.cpp b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.cpp new file mode 100644 index 00000000000..bb9da2d9c7f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.cpp @@ -0,0 +1,29 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "pending_gid_to_lid_changes.h" +#include "gid_to_lid_change_handler.h" + +namespace proton { + +PendingGidToLidChanges::PendingGidToLidChanges(GidToLidChangeHandler& handler, std::vector<PendingGidToLidChange> &&pending_changes) + : IPendingGidToLidChanges(), + _handler(handler), + _pending_changes(std::move(pending_changes)) +{ +} + +PendingGidToLidChanges::~PendingGidToLidChanges() = default; + +void +PendingGidToLidChanges::notify_done() +{ + for (auto& change : _pending_changes) { + if (change.is_remove()) { + _handler.notifyRemoveDone(change.get_gid(), change.get_serial_num()); + } else { + _handler.notifyPutDone(std::move(change).steal_context(), change.get_gid(), change.get_lid(), change.get_serial_num()); + } + } +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.h b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.h new file mode 100644 index 00000000000..a3e74e74b1f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.h @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_pending_gid_to_lid_changes.h" +#include "pending_gid_to_lid_change.h" +#include <vector> + +namespace proton { + +class GidToLidChangeHandler; + +/* + * Class for a vector of gid to lid changes awaiting a force commit. + */ +class PendingGidToLidChanges : public IPendingGidToLidChanges +{ + GidToLidChangeHandler& _handler; + std::vector<PendingGidToLidChange> _pending_changes; +public: + PendingGidToLidChanges(GidToLidChangeHandler& handler, std::vector<PendingGidToLidChange> &&pending_changes); + ~PendingGidToLidChanges() override; + void notify_done() override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp deleted file mode 100644 index e806628bc02..00000000000 --- a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "pending_notify_remove_done.h" -#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> -#include <cassert> - -namespace proton -{ - -PendingNotifyRemoveDone::PendingNotifyRemoveDone() - : _gidToLidChangeHandler(nullptr), - _gid(), - _serialNum(0), - _pending(false) -{ -} - -PendingNotifyRemoveDone::PendingNotifyRemoveDone(PendingNotifyRemoveDone &&rhs) - : _gidToLidChangeHandler(rhs._gidToLidChangeHandler), - _gid(rhs._gid), - _serialNum(rhs._serialNum), - _pending(rhs._pending) -{ - rhs._pending = false; -} - -PendingNotifyRemoveDone::~PendingNotifyRemoveDone() -{ - assert(!_pending); // Fail if notifyRemoveDone is still pending -} - -void -PendingNotifyRemoveDone::setup(IGidToLidChangeHandler &gidToLidChangeHandler, document::GlobalId gid, search::SerialNum serialNum) -{ - _gidToLidChangeHandler = &gidToLidChangeHandler; - _gid = gid; - _serialNum = serialNum; - _pending = true; -} - -void -PendingNotifyRemoveDone::invoke() -{ - if (_pending) { - _gidToLidChangeHandler->notifyRemoveDone(_gid, _serialNum); - _pending = false; - } -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h deleted file mode 100644 index 95aad182b10..00000000000 --- a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/document/base/globalid.h> -#include <vespa/searchlib/common/serialnum.h> - -namespace proton -{ - -class IGidToLidChangeHandler; - -/* - * Class used to keep track of a pending notifyRemoveDone() call to - * a gid to lid change handler. - */ -class PendingNotifyRemoveDone -{ - IGidToLidChangeHandler *_gidToLidChangeHandler; - document::GlobalId _gid; - search::SerialNum _serialNum; - bool _pending; - -public: - PendingNotifyRemoveDone(); - PendingNotifyRemoveDone(PendingNotifyRemoveDone &&rhs); - PendingNotifyRemoveDone(const PendingNotifyRemoveDone &rhs) = delete; - PendingNotifyRemoveDone &operator=(const PendingNotifyRemoveDone &rhs) = delete; - PendingNotifyRemoveDone &operator=(PendingNotifyRemoveDone &&rhs) = delete; - ~PendingNotifyRemoveDone(); - void setup(IGidToLidChangeHandler &gidToLidChangeHandler, document::GlobalId gid, search::SerialNum serialNum); - void invoke(); -}; - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 93432221e61..57445775df3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -80,7 +80,6 @@ vespa_add_library(searchcore_server STATIC pruneremoveddocumentsjob.cpp putdonecontext.cpp reconfig_params.cpp - remove_batch_done_context.cpp remove_operations_rate_tracker.cpp removedonecontext.cpp removedonetask.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp index 9c9c3fc7eca..32554555984 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp @@ -3,6 +3,7 @@ #include "forcecommitcontext.h" #include "forcecommitdonetask.h" #include <vespa/searchcore/proton/common/docid_limit.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <cassert> namespace proton { @@ -10,9 +11,10 @@ namespace proton { ForceCommitContext::ForceCommitContext(vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, PendingLidTrackerBase::Snapshot lidsToCommit, + std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes, std::shared_ptr<IDestructorCallback> onDone) : _executor(executor), - _task(std::make_unique<ForceCommitDoneTask>(documentMetaStore)), + _task(std::make_unique<ForceCommitDoneTask>(documentMetaStore, std::move(pending_gid_to_lid_changes))), _committedDocIdLimit(0u), _docIdLimit(nullptr), _lidsToCommit(std::move(lidsToCommit)), diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h index a9987a15da6..73c4ac97f42 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h @@ -12,6 +12,7 @@ namespace proton { class ForceCommitDoneTask; struct IDocumentMetaStore; class DocIdLimit; +class IPendingGidToLidChanges; /** * Context class for forced commits that schedules a task when @@ -34,6 +35,7 @@ public: ForceCommitContext(vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, PendingLidTrackerBase::Snapshot lidsToCommit, + std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes, std::shared_ptr<IDestructorCallback> onDone); ~ForceCommitContext() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp index 81da2a4ec3e..733c15d55bb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp @@ -2,13 +2,15 @@ #include "forcecommitdonetask.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> namespace proton { -ForceCommitDoneTask::ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore) +ForceCommitDoneTask::ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore, std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes) : _lidsToReuse(), _holdUnblockShrinkLidSpace(false), - _documentMetaStore(documentMetaStore) + _documentMetaStore(documentMetaStore), + _pending_gid_to_lid_changes(std::move(pending_gid_to_lid_changes)) { } @@ -24,6 +26,9 @@ ForceCommitDoneTask::reuseLids(std::vector<uint32_t> &&lids) void ForceCommitDoneTask::run() { + if (_pending_gid_to_lid_changes) { + _pending_gid_to_lid_changes->notify_done(); + } if (!_lidsToReuse.empty()) { if (_lidsToReuse.size() == 1) { _documentMetaStore.removeComplete(_lidsToReuse[0]); diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h index cffe4199e84..fe95d1575e9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h @@ -8,6 +8,7 @@ namespace proton { struct IDocumentMetaStore; +class IPendingGidToLidChanges; /** * Class for task to be executed when a forced commit has completed and @@ -28,9 +29,10 @@ class ForceCommitDoneTask : public vespalib::Executor::Task std::vector<uint32_t> _lidsToReuse; bool _holdUnblockShrinkLidSpace; IDocumentMetaStore &_documentMetaStore; + std::unique_ptr<IPendingGidToLidChanges> _pending_gid_to_lid_changes; public: - ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore); + ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore, std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes); ~ForceCommitDoneTask() override; void reuseLids(std::vector<uint32_t> &&lids); @@ -42,7 +44,7 @@ public: void run() override; bool empty() const { - return _lidsToReuse.empty() && !_holdUnblockShrinkLidSpace; + return _lidsToReuse.empty() && !_holdUnblockShrinkLidSpace && !_pending_gid_to_lid_changes; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp index 20ffe203235..23caaf1250b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp @@ -10,18 +10,12 @@ using document::Document; namespace proton { PutDoneContext::PutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, - search::SerialNum serialNum, bool enableNotifyPut) + uint32_t lid) : OperationDoneContext(std::move(token)), _uncommitted(std::move(uncommitted)), _lid(lid), _docIdLimit(nullptr), - _gidToLidChangeHandler(gidToLidChangeHandler), - _gid(gid), - _serialNum(serialNum), - _enableNotifyPut(enableNotifyPut), _doc(std::move(doc)) { } @@ -31,9 +25,6 @@ PutDoneContext::~PutDoneContext() if (_docIdLimit != nullptr) { _docIdLimit->bumpUpLimit(_lid + 1); } - if (_enableNotifyPut) { - _gidToLidChangeHandler.notifyPutDone(steal(), _gid, _lid, _serialNum); - } } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h index c5e8c558c9e..e7271d8a1b3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h @@ -12,7 +12,6 @@ namespace document { class Document; } namespace proton { class DocIdLimit; -class IGidToLidChangeHandler; /** * Context class for document put operations that acks operation when @@ -26,16 +25,12 @@ class PutDoneContext : public OperationDoneContext IPendingLidTracker::Token _uncommitted; uint32_t _lid; DocIdLimit *_docIdLimit; - IGidToLidChangeHandler &_gidToLidChangeHandler; - document::GlobalId _gid; - search::SerialNum _serialNum; - bool _enableNotifyPut; std::shared_ptr<const document::Document> _doc; public: - PutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, IGidToLidChangeHandler &gidToLidChangeHandler, + PutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, std::shared_ptr<const document::Document> doc, - const document::GlobalId &gid, uint32_t lid, search::SerialNum serialNum, bool enableNotifyPut); + uint32_t lid); ~PutDoneContext() override; void registerPutLid(DocIdLimit *docIdLimit) { _docIdLimit = docIdLimit; } diff --git a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.cpp b/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.cpp deleted file mode 100644 index b0ece5f35a1..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "remove_batch_done_context.h" -#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> - -namespace proton { - -RemoveBatchDoneContext::RemoveBatchDoneContext(vespalib::Executor &executor, - vespalib::Executor::Task::UP task, - IGidToLidChangeHandler &gidToLidChangeHandler, - std::vector<document::GlobalId> gidsToRemove, - search::SerialNum serialNum) - : search::ScheduleTaskCallback(executor, std::move(task)), - _gidToLidChangeHandler(gidToLidChangeHandler), - _gidsToRemove(std::move(gidsToRemove)), - _serialNum(serialNum) -{ -} - -RemoveBatchDoneContext::~RemoveBatchDoneContext() -{ - for (const auto &gid : _gidsToRemove) { - _gidToLidChangeHandler.notifyRemoveDone(gid, _serialNum); - } -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.h b/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.h deleted file mode 100644 index 2a93239574a..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/searchlib/common/scheduletaskcallback.h> -#include <vespa/document/base/globalid.h> -#include <vespa/searchlib/common/serialnum.h> -#include <vector> - -namespace proton -{ - -class IGidToLidChangeHandler; - -/** - * Context class for document batch remove that notifies gid to lid - * change handler about each remove done and schedules a - * task when instance is destroyed. Typically a shared pointer to an - * instance is passed around to multiple worker threads that performs - * portions of a larger task before dropping the shared pointer. - */ -class RemoveBatchDoneContext : public search::ScheduleTaskCallback -{ - IGidToLidChangeHandler &_gidToLidChangeHandler; - std::vector<document::GlobalId> _gidsToRemove; - search::SerialNum _serialNum; - -public: - RemoveBatchDoneContext(vespalib::Executor &executor, - vespalib::Executor::Task::UP task, - IGidToLidChangeHandler &gidToLidChangeHandler, - std::vector<document::GlobalId> gidsToRemove, - search::SerialNum serialNum); - - virtual ~RemoveBatchDoneContext(); -}; - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp index d35e6bb7127..859d8693f6d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp @@ -9,11 +9,10 @@ namespace proton { RemoveDoneContext::RemoveDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, uint32_t lid) + uint32_t lid) : OperationDoneContext(std::move(token)), _executor(executor), _task(), - _pendingNotifyRemoveDone(std::move(pendingNotifyRemoveDone)), _uncommitted(std::move(uncommitted)) { if (lid != 0) { @@ -23,7 +22,6 @@ RemoveDoneContext::RemoveDoneContext(FeedToken token, IPendingLidTracker::Token RemoveDoneContext::~RemoveDoneContext() { - _pendingNotifyRemoveDone.invoke(); ack(); if (_task) { vespalib::Executor::Task::UP res = _executor.execute(std::move(_task)); diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h index 7b6c6be1fe1..485b82dd141 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h @@ -3,7 +3,6 @@ #pragma once #include "operationdonecontext.h" -#include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> #include <vespa/searchcore/proton/common/ipendinglidtracker.h> #include <vespa/vespalib/util/executor.h> #include <vespa/document/base/globalid.h> @@ -26,12 +25,11 @@ class RemoveDoneContext : public OperationDoneContext { vespalib::Executor &_executor; std::unique_ptr<vespalib::Executor::Task> _task; - PendingNotifyRemoveDone _pendingNotifyRemoveDone; IPendingLidTracker::Token _uncommitted; public: RemoveDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, uint32_t lid); + uint32_t lid); ~RemoveDoneContext() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index bf357188766..7c3c796bda3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -5,7 +5,6 @@ #include "ireplayconfig.h" #include "operationdonecontext.h" #include "putdonecontext.h" -#include "remove_batch_done_context.h" #include "removedonecontext.h" #include "updatedonecontext.h" #include <vespa/document/datatype/documenttype.h> @@ -15,7 +14,9 @@ #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <vespa/searchlib/common/gatecallback.h> +#include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/exceptions.h> @@ -48,11 +49,10 @@ private: public: PutDoneContextForMove(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, search::SerialNum serialNum, - bool enableNotifyPut, IDestructorCallback::SP moveDoneCtx) - : PutDoneContext(std::move(token), std::move(uncommitted), gidToLidChangeHandler, std::move(doc), gid, lid, serialNum, enableNotifyPut), + uint32_t lid, + IDestructorCallback::SP moveDoneCtx) + : PutDoneContext(std::move(token), std::move(uncommitted),std::move(doc), lid), _moveDoneCtx(std::move(moveDoneCtx)) {} ~PutDoneContextForMove() override = default; @@ -60,31 +60,28 @@ public: std::shared_ptr<PutDoneContext> createPutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, - SerialNum serialNum, bool enableNotifyPut, + uint32_t lid, IDestructorCallback::SP moveDoneCtx) { std::shared_ptr<PutDoneContext> result; if (moveDoneCtx) { - result = std::make_shared<PutDoneContextForMove>(std::move(token), std::move(uncommitted), gidToLidChangeHandler, - std::move(doc), gid, lid, serialNum, enableNotifyPut, std::move(moveDoneCtx)); + result = std::make_shared<PutDoneContextForMove>(std::move(token), std::move(uncommitted), + std::move(doc), lid, std::move(moveDoneCtx)); } else { - result = std::make_shared<PutDoneContext>(std::move(token), std::move(uncommitted), gidToLidChangeHandler, - std::move(doc), gid, lid, serialNum, enableNotifyPut); + result = std::make_shared<PutDoneContext>(std::move(token), std::move(uncommitted), + std::move(doc), lid); } return result; } std::shared_ptr<PutDoneContext> createPutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, SerialNum serialNum, bool enableNotifyPut) + uint32_t lid) { - return createPutDoneContext(std::move(token), std::move(uncommitted), gidToLidChangeHandler, std::move(doc), gid, - lid, serialNum, enableNotifyPut, IDestructorCallback::SP()); + return createPutDoneContext(std::move(token), std::move(uncommitted), std::move(doc), + lid, IDestructorCallback::SP()); } std::shared_ptr<UpdateDoneContext> @@ -109,10 +106,10 @@ private: public: RemoveDoneContextForMove(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, - IDocumentMetaStore &documentMetaStore, PendingNotifyRemoveDone &&pendingNotifyRemoveDone, + IDocumentMetaStore &documentMetaStore, uint32_t lid, IDestructorCallback::SP moveDoneCtx) : RemoveDoneContext(std::move(token), std::move(uncommitted), executor, - documentMetaStore, std::move(pendingNotifyRemoveDone) ,lid), + documentMetaStore, lid), _moveDoneCtx(std::move(moveDoneCtx)) {} ~RemoveDoneContextForMove() override = default; @@ -120,16 +117,16 @@ public: std::shared_ptr<RemoveDoneContext> createRemoveDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, - IDocumentMetaStore &documentMetaStore,PendingNotifyRemoveDone &&pendingNotifyRemoveDone, + IDocumentMetaStore &documentMetaStore, uint32_t lid, IDestructorCallback::SP moveDoneCtx) { if (moveDoneCtx) { return std::make_shared<RemoveDoneContextForMove> - (std::move(token), std::move(uncommitted), executor, documentMetaStore, std::move(pendingNotifyRemoveDone), + (std::move(token), std::move(uncommitted), executor, documentMetaStore, lid, std::move(moveDoneCtx)); } else { return std::make_shared<RemoveDoneContext> - (std::move(token), std::move(uncommitted), executor, documentMetaStore, std::move(pendingNotifyRemoveDone), lid); + (std::move(token), std::move(uncommitted), executor, documentMetaStore, lid); } } @@ -243,6 +240,7 @@ StoreOnlyFeedView::forceCommit(SerialNum serialNum, DoneCallback onDone) { internalForceCommit(serialNum, std::make_shared<ForceCommitContext>(_writeService.master(), _metaStore, _pendingLidsForCommit->produceSnapshot(), + _gidToLidChangeHandler.grab_pending_changes(), std::move(onDone))); } @@ -305,17 +303,18 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) putOp.getSubDbId(), putOp.getLid(), putOp.getPrevSubDbId(), putOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId.getGlobalId(), docId); + adjustMetaStore(putOp, docId.getGlobalId(), docId); auto uncommitted = get_pending_lid_token(putOp); bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - const document::GlobalId &gid = docId.getGlobalId(); + if (putOp.changedDbdId() && useDocumentMetaStore(serialNum)) { + _gidToLidChangeHandler.notifyPut(token, docId.getGlobalId(), putOp.getLid(), serialNum); + } std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), - _gidToLidChangeHandler, doc, gid, putOp.getLid(), serialNum, - putOp.changedDbdId() && useDocumentMetaStore(serialNum)); + doc, putOp.getLid()); putSummary(serialNum, putOp.getLid(), doc, onWriteDone); putAttributes(serialNum, putOp.getLid(), *doc, onWriteDone); putIndexedFields(serialNum, putOp.getLid(), doc, onWriteDone); @@ -323,7 +322,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) if (docAlreadyExists && putOp.changedDbdId()) { assert(!putOp.getValidDbdId(_params._subDbId)); internalRemove(std::move(token), _pendingLidsForCommit->produce(putOp.getPrevLid()), serialNum, - std::move(pendingNotifyRemoveDone), putOp.getPrevLid(), IDestructorCallback::SP()); + putOp.getPrevLid(), IDestructorCallback::SP()); } } @@ -574,7 +573,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI _params._docTypeName.toString().c_str(), serialNum, docId.toString().c_str(), rmOp.getSubDbId(), rmOp.getLid(), rmOp.getPrevSubDbId(), rmOp.getPrevLid(), _params._subDbId); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId.getGlobalId(), docId); + adjustMetaStore(rmOp, docId.getGlobalId(), docId); auto uncommitted = get_pending_lid_token(rmOp); if (rmOp.getValidDbdId(_params._subDbId)) { @@ -587,7 +586,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI if (rmOp.changedDbdId()) { assert(!rmOp.getValidDbdId(_params._subDbId)); internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, - std::move(pendingNotifyRemoveDone), rmOp.getPrevLid(), IDestructorCallback::SP()); + rmOp.getPrevLid(), IDestructorCallback::SP()); } } } @@ -599,13 +598,13 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid assert(rmOp.notMovingLidInSameSubDb()); const SerialNum serialNum = rmOp.getSerialNum(); DocumentId dummy; - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, rmOp.getGlobalId(), dummy); + adjustMetaStore(rmOp, rmOp.getGlobalId(), dummy); auto uncommitted = _pendingLidsForCommit->produce(rmOp.getLid()); if (rmOp.getValidPrevDbdId(_params._subDbId)) { if (rmOp.changedDbdId()) { assert(!rmOp.getValidDbdId(_params._subDbId)); - internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, std::move(pendingNotifyRemoveDone), + internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, rmOp.getPrevLid(), IDestructorCallback::SP()); } } @@ -613,23 +612,22 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid void StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token uncommitted, SerialNum serialNum, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid, + Lid lid, IDestructorCallback::SP moveDoneCtx) { bool explicitReuseLid = _lidReuseDelayer.delayReuse(lid); std::shared_ptr<RemoveDoneContext> onWriteDone; onWriteDone = createRemoveDoneContext(std::move(token), std::move(uncommitted),_writeService.master(), _metaStore, - std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), + (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); removeAttributes(serialNum, lid, onWriteDone); removeIndexedFields(serialNum, lid, onWriteDone); } -PendingNotifyRemoveDone +void StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & gid, const DocumentId &docId) { - PendingNotifyRemoveDone pendingNotifyRemoveDone; const SerialNum serialNum = op.getSerialNum(); if (useDocumentMetaStore(serialNum)) { if (op.getValidDbdId(_params._subDbId)) { @@ -644,13 +642,11 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & } else if (op.getValidPrevDbdId(_params._subDbId)) { vespalib::Gate gate; _gidToLidChangeHandler.notifyRemove(std::make_shared<search::GateCallback>(gate), gid, serialNum); - pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, gid, serialNum); gate.await(); removeMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED); } _metaStore.commit(serialNum, serialNum); } - return pendingNotifyRemoveDone; } void @@ -695,8 +691,7 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo } else { removeBatchDoneTask = makeLambdaTask([]() {}); } - onWriteDone = std::make_shared<RemoveBatchDoneContext>(_writeService.master(), std::move(removeBatchDoneTask), - _gidToLidChangeHandler, std::move(gidsToRemove), serialNum); + onWriteDone = std::make_shared<search::ScheduleTaskCallback>(_writeService.master(), std::move(removeBatchDoneTask)); if (remove_index_and_attributes) { removeIndexedFields(serialNum, lidsToRemove, onWriteDone); removeAttributes(serialNum, lidsToRemove, onWriteDone); @@ -767,20 +762,21 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: moveOp.getSubDbId(), moveOp.getLid(), moveOp.getPrevSubDbId(), moveOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); + adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - const document::GlobalId &gid = docId.getGlobalId(); + if (moveOp.changedDbdId() && useDocumentMetaStore(serialNum)) { + _gidToLidChangeHandler.notifyPut(FeedToken(), docId.getGlobalId(), moveOp.getLid(), serialNum); + } std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), - _gidToLidChangeHandler, doc, gid, moveOp.getLid(), serialNum, - moveOp.changedDbdId() && useDocumentMetaStore(serialNum), doneCtx); + doc, moveOp.getLid(), doneCtx); putSummary(serialNum, moveOp.getLid(), doc, onWriteDone); putAttributes(serialNum, moveOp.getLid(), *doc, onWriteDone); putIndexedFields(serialNum, moveOp.getLid(), doc, onWriteDone); } if (docAlreadyExists && moveOp.changedDbdId()) { - internalRemove(FeedToken(), _pendingLidsForCommit->produce(moveOp.getPrevLid()), serialNum, std::move(pendingNotifyRemoveDone), moveOp.getPrevLid(), doneCtx); + internalRemove(FeedToken(), _pendingLidsForCommit->produce(moveOp.getPrevLid()), serialNum, moveOp.getPrevLid(), doneCtx); } } @@ -820,6 +816,7 @@ StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) getDocumentMetaStore()->get().compactLidSpace(op.getLidLimit()); auto commitContext(std::make_shared<ForceCommitContext>(_writeService.master(), _metaStore, _pendingLidsForCommit->produceSnapshot(), + _gidToLidChangeHandler.grab_pending_changes(), DoneCallback())); commitContext->holdUnblockShrinkLidSpace(); internalForceCommit(serialNum, commitContext); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index da7d5e53a88..9927c93add4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -16,7 +16,6 @@ #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> #include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> #include <vespa/searchcore/proton/persistenceengine/resulthandler.h> -#include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/query/base.h> #include <vespa/vespalib/util/threadstackexecutorbase.h> @@ -169,7 +168,7 @@ private: return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum; } - PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::GlobalId & gid, const document::DocumentId &docId); + void adjustMetaStore(const DocumentOperation &op, const document::GlobalId & gid, const document::DocumentId &docId); void internalPut(FeedToken token, const PutOperation &putOp); void internalUpdate(FeedToken token, const UpdateOperation &updOp); @@ -182,7 +181,6 @@ private: size_t removeDocuments(const RemoveDocumentsOperation &op, bool remove_index_and_attribute_fields); void internalRemove(FeedToken token, IPendingLidTracker::Token uncommitted, SerialNum serialNum, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid, std::shared_ptr<search::IDestructorCallback> moveDoneCtx); IPendingLidTracker::Token get_pending_lid_token(const DocumentOperation &op); diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index f9531486e9b..976cdc70048 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -3,6 +3,7 @@ #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -42,9 +43,9 @@ public: _removes.emplace_back(docTypeName, keepNames); } - void notifyPutDone(IDestructorCallbackSP, document::GlobalId, uint32_t, SerialNum) override { } + void notifyPut(IDestructorCallbackSP, document::GlobalId, uint32_t, SerialNum) override { } void notifyRemove(IDestructorCallbackSP, document::GlobalId, SerialNum) override { } - void notifyRemoveDone(document::GlobalId, SerialNum) override { } + std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() override { return {}; } void assertAdds(const std::vector<AddEntry> &expAdds) { diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index da77e29dbb0..daa85c91b2c 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -49,6 +49,7 @@ using search::tensor::NearestNeighborIndexSaver; using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using vespalib::eval::TensorSpec; +using vespalib::eval::CellType; using vespalib::eval::ValueType; using vespalib::eval::Value; using vespalib::eval::EngineOrFactory; @@ -228,11 +229,11 @@ class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory { std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, size_t vector_size, - ValueType::CellType cell_type, + CellType cell_type, const search::attribute::HnswIndexParams& params) const override { (void) vector_size; (void) params; - assert(cell_type == ValueType::CellType::DOUBLE); + assert(cell_type == CellType::DOUBLE); return std::make_unique<MockNearestNeighborIndex>(vectors); } }; diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index f6ca0bd1427..23cb3831b6d 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -25,7 +25,7 @@ using search::AttributeVector; using search::BitVector; using vespalib::eval::Value; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; using vespalib::eval::TensorSpec; using vespalib::eval::EngineOrFactory; using search::tensor::DistanceFunction; diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index a9e24e056f2..06fb95089fd 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -33,7 +33,7 @@ void verify_geo_miles(const DistanceFunction *dist_fun, TEST(DistanceFunctionsTest, euclidean_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto euclid = make_distance_function(DistanceMetric::Euclidean, ct); @@ -54,7 +54,7 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) TEST(DistanceFunctionsTest, angular_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto angular = make_distance_function(DistanceMetric::Angular, ct); @@ -109,7 +109,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto innerproduct = make_distance_function(DistanceMetric::InnerProduct, ct); @@ -144,7 +144,7 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) TEST(DistanceFunctionsTest, hamming_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto hamming = make_distance_function(DistanceMetric::Hamming, ct); @@ -184,7 +184,7 @@ TEST(DistanceFunctionsTest, hamming_gives_expected_score) TEST(GeoDegreesTest, gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto geodeg = make_distance_function(DistanceMetric::GeoDegrees, ct); std::vector<double> g1_sfo{37.61, -122.38}; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index dd685ce5c43..85b7e8f89e8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -7,8 +7,7 @@ using search::tensor::DenseTensorAttribute; using vespalib::ConstArrayRef; using vespalib::tensor::MutableDenseTensorView; using vespalib::eval::TypedCells; - -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; namespace search::queryeval { diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp index 0bb6f339455..aca14a1575e 100644 --- a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp @@ -28,7 +28,7 @@ make_random_level_generator(uint32_t m) std::unique_ptr<NearestNeighborIndex> DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors, size_t vector_size, - vespalib::eval::ValueType::CellType cell_type, + vespalib::eval::CellType cell_type, const search::attribute::HnswIndexParams& params) const { (void) vector_size; diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h index 6a9ded92b60..67a19a5431a 100644 --- a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h @@ -13,7 +13,7 @@ class DefaultNearestNeighborIndexFactory : public NearestNeighborIndexFactory { public: std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, size_t vector_size, - vespalib::eval::ValueType::CellType cell_type, + vespalib::eval::CellType cell_type, const search::attribute::HnswIndexParams& params) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index 1abc3800d97..ddbb956838b 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -9,7 +9,7 @@ using vespalib::datastore::Handle; using vespalib::tensor::MutableDenseTensorView; using vespalib::eval::Value; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using CellType = vespalib::eval::CellType; namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index a868dfe191b..81b27b56258 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -4,44 +4,45 @@ #include "distance_functions.h" using search::attribute::DistanceMetric; +using vespalib::eval::CellType; using vespalib::eval::ValueType; namespace search::tensor { DistanceFunction::UP -make_distance_function(DistanceMetric variant, ValueType::CellType cell_type) +make_distance_function(DistanceMetric variant, CellType cell_type) { switch (variant) { case DistanceMetric::Euclidean: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<SquaredEuclideanDistance<float>>(); } else { return std::make_unique<SquaredEuclideanDistance<double>>(); } break; case DistanceMetric::Angular: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<AngularDistance<float>>(); } else { return std::make_unique<AngularDistance<double>>(); } break; case DistanceMetric::GeoDegrees: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<GeoDegreesDistance<float>>(); } else { return std::make_unique<GeoDegreesDistance<double>>(); } break; case DistanceMetric::InnerProduct: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<InnerProductDistance<float>>(); } else { return std::make_unique<InnerProductDistance<double>>(); } break; case DistanceMetric::Hamming: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<HammingDistance<float>>(); } else { return std::make_unique<HammingDistance<double>>(); diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h index c86e40279bc..abb1f503694 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h @@ -14,6 +14,6 @@ namespace search::tensor { **/ DistanceFunction::UP make_distance_function(search::attribute::DistanceMetric variant, - vespalib::eval::ValueType::CellType cell_type); + vespalib::eval::CellType cell_type); } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h index 089119944a7..e5c15266ceb 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h @@ -20,7 +20,7 @@ public: virtual ~NearestNeighborIndexFactory() {} virtual std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, size_t vector_size, - vespalib::eval::ValueType::CellType cell_type, + vespalib::eval::CellType cell_type, const search::attribute::HnswIndexParams& params) const = 0; }; diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java index a2b82978a26..e0e3469e257 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java @@ -29,6 +29,7 @@ public class MetricReceiver { private volatile Map<String, MetricSettings> metricSettings; private static final class NullCounter extends Counter { + NullCounter() { super(null, null, null); } @@ -72,18 +73,23 @@ public class MetricReceiver { public PointBuilder builder() { return super.builder(); } + } public static final class MockReceiver extends MetricReceiver { + private final ThreadLocalDirectory<Bucket, Sample> collection; + private MockReceiver(ThreadLocalDirectory<Bucket, Sample> collection) { super(collection, null); this.collection = collection; } + public MockReceiver() { this(new ThreadLocalDirectory<>(new MetricUpdater())); } - /** gathers all data since last snapshot */ + + /** Gathers all data since last snapshot */ public Bucket getSnapshot() { final Bucket merged = new Bucket(); for (Bucket b : collection.fetch()) { @@ -91,13 +97,16 @@ public class MetricReceiver { } return merged; } - /** utility method for testing */ + + /** Utility method for testing */ public Point point(String dim, String val) { return pointBuilder().set(dim, val).build(); } + } private static final class NullReceiver extends MetricReceiver { + NullReceiver() { super(null, null); } @@ -164,21 +173,18 @@ public class MetricReceiver { * {@link #declareGauge(String)}, or {@link #declareGauge(String, Point)} * instead. * - * @param s - * a single simple containing all meta data necessary to update a - * metric + * @param sample a single simple containing all meta data necessary to update a metric */ - public void update(Sample s) { + public void update(Sample sample) { // pass around the receiver instead of histogram settings to avoid reading any volatile if unnecessary - s.setReceiver(this); - metricsCollection.update(s); + sample.setReceiver(this); + metricsCollection.update(sample); } /** * Declare a counter metric without setting any default position. * - * @param name - * the name of the metric + * @param name the name of the metric * @return a thread-safe counter */ public Counter declareCounter(String name) { @@ -189,11 +195,8 @@ public class MetricReceiver { * Declare a counter metric, with default dimension values as given. Create * the point argument by using a builder from {@link #pointBuilder()}. * - * @param name - * the name of the metric - * @param boundDimensions - * dimensions which have a fixed value in the life cycle of the - * metric object or null + * @param name the name of the metric + * @param boundDimensions dimensions which have a fixed value in the life cycle of the metric object or null * @return a thread-safe counter with given default values */ public Counter declareCounter(String name, Point boundDimensions) { @@ -203,8 +206,7 @@ public class MetricReceiver { /** * Declare a gauge metric with any default position. * - * @param name - * the name of the metric + * @param name the name of the metric * @return a thread-safe gauge instance */ public Gauge declareGauge(String name) { @@ -215,21 +217,12 @@ public class MetricReceiver { * Declare a gauge metric, with default dimension values as given. Create * the point argument by using a builder from {@link #pointBuilder()}. * - * @param name - * the name of the metric - * @param boundDimensions - * dimensions which have a fixed value in the life cycle of the - * metric object or null + * @param name the name of the metric + * @param boundDimensions dimensions which have a fixed value in the life cycle of the metric object or null * @return a thread-safe gauge metric */ public Gauge declareGauge(String name, Point boundDimensions) { - Optional<Point> optionalOfBoundDimensions; - if (boundDimensions == null) { - optionalOfBoundDimensions = Optional.empty(); - } else { - optionalOfBoundDimensions = Optional.of(boundDimensions); - } - return declareGauge(name, optionalOfBoundDimensions, null); + return declareGauge(name, Optional.ofNullable(boundDimensions), null); } /** @@ -238,13 +231,9 @@ public class MetricReceiver { * MetricSettings instances are built using * {@link MetricSettings.Builder}. * - * @param name - * the name of the metric - * @param boundDimensions - * an optional of dimensions which have a fixed value in the life - * cycle of the metric object - * @param customSettings - * any optional settings + * @param name the name of the metric + * @param boundDimensions an optional of dimensions which have a fixed value in the life cycle of the metric object + * @param customSettings any optional settings * @return a thread-safe gauge metric */ public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { @@ -283,14 +272,8 @@ public class MetricReceiver { /** * Add how to build a histogram for a given metric. * - * <p> - * Do note, this is not part of the public API. - * </p> - * - * @param metricName - * the metric where samples should be put in a histogram - * @param definition - * settings for a histogram + * @param metricName the metric where samples should be put in a histogram + * @param definition settings for a histogram */ void addMetricDefinition(String metricName, MetricSettings definition) { synchronized (histogramDefinitionsLock) { @@ -304,15 +287,9 @@ public class MetricReceiver { } /** - * Get how to build a histogram for a given metric, or null if no histogram - * should be created. - * - * <p> - * Do note, this is not part of the public API. - * </p> + * Get how to build a histogram for a given metric, or null if no histogram should be created. * - * @param metricName - * the name of an arbitrary metric + * @param metricName the name of an arbitrary metric * @return the corresponding histogram definition or null */ MetricSettings getMetricDefinition(String metricName) { diff --git a/storage/src/tests/persistence/common/filestortestfixture.cpp b/storage/src/tests/persistence/common/filestortestfixture.cpp index c6a0bdbb540..079abd6df06 100644 --- a/storage/src/tests/persistence/common/filestortestfixture.cpp +++ b/storage/src/tests/persistence/common/filestortestfixture.cpp @@ -2,6 +2,7 @@ #include <vespa/storage/persistence/messages.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <tests/persistence/common/filestortestfixture.h> #include <vespa/document/repo/documenttyperepo.h> @@ -15,7 +16,6 @@ using document::test::makeDocumentBucket; namespace storage { -spi::LoadType FileStorTestFixture::defaultLoadType = spi::LoadType(0, "default"); const uint32_t FileStorTestFixture::MSG_WAIT_TIME; void @@ -51,10 +51,8 @@ FileStorTestFixture::TearDown() void FileStorTestFixture::createBucket(const document::BucketId& bid) { - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - _node->getPersistenceProvider().createBucket( - makeSpiBucket(bid), context); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); + _node->getPersistenceProvider().createBucket(makeSpiBucket(bid), context); StorBucketDatabase::WrappedEntry entry( _node->getStorageBucketDatabase().get(bid, "foo", diff --git a/storage/src/tests/persistence/common/filestortestfixture.h b/storage/src/tests/persistence/common/filestortestfixture.h index dcfeb42b4fd..69866f7bbf2 100644 --- a/storage/src/tests/persistence/common/filestortestfixture.h +++ b/storage/src/tests/persistence/common/filestortestfixture.h @@ -15,8 +15,6 @@ namespace storage { class FileStorTestFixture : public ::testing::Test { public: - static spi::LoadType defaultLoadType; - std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<vdstestlib::DirConfig> _config; const document::DocumentType* _testdoctype1; diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 7cfe7ea3761..3cac188cb17 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -56,7 +56,7 @@ namespace storage { namespace { -spi::LoadType defaultLoadType(0, "default"); +metrics::LoadType defaultLoadType(0, "default"); struct TestFileStorComponents; @@ -86,7 +86,7 @@ struct FileStorTestBase : Test { void TearDown() override; void createBucket(document::BucketId bid) { - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); _node->getPersistenceProvider().createBucket(makeSpiBucket(bid), context); StorBucketDatabase::WrappedEntry entry( @@ -728,7 +728,7 @@ TEST_F(FileStorManagerTest, priority) { for (uint32_t i=0; i<documents.size(); ++i) { document::BucketId bucket(16, factory.getBucketId(documents[i]->getId()).getRawId()); - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); _node->getPersistenceProvider().createBucket(makeSpiBucket(bucket), context); } @@ -790,8 +790,7 @@ TEST_F(FileStorManagerTest, split1) { documents.push_back(doc); } document::BucketIdFactory factory; - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); { // Populate bucket with the given data for (uint32_t i=0; i<documents.size(); ++i) { @@ -905,7 +904,7 @@ TEST_F(FileStorManagerTest, split_single_group) { auto& top = c.top; setClusterState("storage:2 distributor:1"); - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); for (uint32_t j=0; j<1; ++j) { // Test this twice, once where all the data ends up in file with // splitbit set, and once where all the data ends up in file with @@ -984,8 +983,7 @@ FileStorTestBase::putDoc(DummyStorageLink& top, uint32_t docNum) { api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); document::BucketIdFactory factory; document::DocumentId docId(vespalib::make_string("id:ns:testdoctype1:n=%" PRIu64 ":%d", target.getId(), docNum)); document::BucketId bucket(16, factory.getBucketId(docId).getRawId()); diff --git a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp index 35aee60d30f..8b38083b33d 100644 --- a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp +++ b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp @@ -2,7 +2,7 @@ #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/persistence/messages.h> -#include <tests/persistence/common/persistenceproviderwrapper.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <tests/persistence/common/filestortestfixture.h> #include <vector> diff --git a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp index eb7f2f883ab..95e2d8e2c43 100644 --- a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp +++ b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/storage/persistence/messages.h> +#include <vespa/storageapi/message/bucket.h> #include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <tests/persistence/common/filestortestfixture.h> @@ -9,7 +10,6 @@ #include <vespa/vespalib/util/thread.h> #include <vespa/vespalib/stllike/hash_set_insert.hpp> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/util/time.h> #include <thread> #include <vespa/log/log.h> @@ -72,7 +72,7 @@ public: } }; -spi::LoadType defaultLoadType(0, "default"); +metrics::LoadType defaultLoadType(0, "default"); } diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 81f98136575..c1da8e9ec06 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -2,6 +2,7 @@ #include <vespa/document/base/testdocman.h> #include <vespa/storage/persistence/mergehandler.h> +#include <vespa/storage/persistence/filestorage/mergestatus.h> #include <tests/persistence/persistencetestutils.h> #include <tests/persistence/common/persistenceproviderwrapper.h> #include <tests/common/message_sender_stub.h> @@ -176,7 +177,7 @@ MergeHandlerTest::HandleApplyBucketDiffReplyInvoker::~HandleApplyBucketDiffReply void MergeHandlerTest::SetUp() { - _context.reset(new spi::Context(documentapi::LoadType::DEFAULT, 0, 0)); + _context = std::make_unique<spi::Context>(0, 0); SingleDiskPersistenceTestUtils::SetUp(); _location = 1234; @@ -1066,7 +1067,7 @@ TEST_F(MergeHandlerTest, apply_bucket_diff_reply_spi_failures) { TEST_F(MergeHandlerTest, remove_from_diff) { framework::defaultimplementation::FakeClock clock; - MergeStatus status(clock, documentapi::LoadType::DEFAULT, 0, 0); + MergeStatus status(clock, 0, 0); std::vector<api::GetBucketDiffCommand::Entry> diff(2); diff[0]._timestamp = 1234; diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index 47e516a1dcc..278cb36b50c 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -26,26 +26,25 @@ namespace storage { namespace { - spi::LoadType defaultLoadType(0, "default"); - - vdstestlib::DirConfig initialize(const std::string & rootOfRoot) { - vdstestlib::DirConfig config(getStandardConfig(true, rootOfRoot)); - std::string rootFolder = getRootFolder(config); - vespalib::rmdir(rootFolder, true); - vespalib::mkdir(vespalib::make_string("%s/disks/d0", rootFolder.c_str()), true); - return config; - } +vdstestlib::DirConfig initialize(const std::string & rootOfRoot) { + vdstestlib::DirConfig config(getStandardConfig(true, rootOfRoot)); + std::string rootFolder = getRootFolder(config); + vespalib::rmdir(rootFolder, true); + vespalib::mkdir(vespalib::make_string("%s/disks/d0", rootFolder.c_str()), true); + return config; +} + +template<typename T> +struct ConfigReader : public T::Subscriber +{ + T config; - template<typename T> - struct ConfigReader : public T::Subscriber - { - T config; + ConfigReader(const std::string& configId) { + T::subscribe(configId, *this); + } + void configure(const T& c) { config = c; } +}; - ConfigReader(const std::string& configId) { - T::subscribe(configId, *this); - } - void configure(const T& c) { config = c; } - }; } PersistenceTestEnvironment::PersistenceTestEnvironment(const std::string & rootOfRoot) @@ -156,7 +155,7 @@ PersistenceTestUtils::doPutOnDisk( { document::Document::SP doc(createRandomDocumentAtLocation(location, timestamp, minSize, maxSize)); spi::Bucket b(makeSpiBucket(document::BucketId(16, location))); - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); getPersistenceProvider().createBucket(b, context); getPersistenceProvider().put(spi::Bucket(b), timestamp, doc, context); return doc; @@ -169,7 +168,7 @@ PersistenceTestUtils::doRemoveOnDisk( spi::Timestamp timestamp, bool persistRemove) { - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); if (persistRemove) { spi::RemoveResult result = getPersistenceProvider().removeIfFound(makeSpiBucket(bucketId),timestamp, docId, context); return result.wasFound(); @@ -185,7 +184,7 @@ PersistenceTestUtils::doUnrevertableRemoveOnDisk( const document::DocumentId& docId, spi::Timestamp timestamp) { - spi::Context context(defaultLoadType, spi::Priority(0),spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0),spi::Trace::TraceLevel(0)); spi::RemoveResult result = getPersistenceProvider().remove(makeSpiBucket(bucketId), timestamp, docId, context); return result.wasFound(); } @@ -194,7 +193,7 @@ spi::GetResult PersistenceTestUtils::doGetOnDisk(const document::BucketId& bucketId, const document::DocumentId& docId) { auto fieldSet = std::make_unique<document::AllFields>(); - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); return getPersistenceProvider().get(makeSpiBucket(bucketId), *fieldSet, docId, context); } @@ -234,7 +233,7 @@ void PersistenceTestUtils::doPut(const document::Document::SP& doc, document::BucketId bid, spi::Timestamp time) { spi::Bucket b(makeSpiBucket(bid)); - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); getPersistenceProvider().createBucket(b, context); getPersistenceProvider().put(b, time, std::move(doc), context); } @@ -244,7 +243,7 @@ PersistenceTestUtils::doUpdate(document::BucketId bid, const document::DocumentUpdate::SP& update, spi::Timestamp time) { - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); return getPersistenceProvider().update(makeSpiBucket(bid), time, update, context); } @@ -255,7 +254,7 @@ PersistenceTestUtils::doRemove(const document::DocumentId& id, spi::Timestamp ti document::BucketId bucket( _env->_component.getBucketIdFactory().getBucketId(id)); bucket.setUsedBits(usedBits); - spi::Context context(defaultLoadType, spi::Priority(0), spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); if (unrevertableRemove) { getPersistenceProvider().remove( makeSpiBucket(bucket), time, id, context); diff --git a/storage/src/tests/persistence/persistencethread_splittest.cpp b/storage/src/tests/persistence/persistencethread_splittest.cpp index 3dd8075176d..45b9f5aeaac 100644 --- a/storage/src/tests/persistence/persistencethread_splittest.cpp +++ b/storage/src/tests/persistence/persistencethread_splittest.cpp @@ -11,9 +11,6 @@ using document::test::makeDocumentBucket; using namespace ::testing; namespace storage { -namespace { -spi::LoadType defaultLoadType(0, "default"); -} struct PersistenceThreadSplitTest : public SingleDiskPersistenceTestUtils { enum SplitCase { @@ -182,8 +179,7 @@ PersistenceThreadSplitTest::doTest(SplitCase splitCase) uint64_t location = 0; uint64_t splitMask = 1ULL << (splitLevelToDivide - 1); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); + spi::Context context(spi::Priority(0), spi::Trace::TraceLevel(0)); spi::Bucket bucket(makeSpiBucket(document::BucketId(currentSplitLevel, 1))); spi::PersistenceProvider& spi(getPersistenceProvider()); spi.deleteBucket(bucket, context); diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp index 69f8268ff2c..4982383685a 100644 --- a/storage/src/tests/persistence/splitbitdetectortest.cpp +++ b/storage/src/tests/persistence/splitbitdetectortest.cpp @@ -16,10 +16,6 @@ using namespace ::testing; namespace storage { -namespace { -spi::LoadType defaultLoadType(0, "default"); -} - struct SplitBitDetectorTest : Test { document::TestDocMan testDocMan; spi::dummy::DummyPersistence provider; @@ -30,8 +26,7 @@ struct SplitBitDetectorTest : Test { : testDocMan(), provider(testDocMan.getTypeRepoSP()), bucket(makeSpiBucket(document::BucketId(1, 1))), - context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)) + context(spi::Priority(0), spi::Trace::TraceLevel(0)) { provider.initialize(); provider.createBucket(bucket, context); diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index a952e227b70..4b509033d2e 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -41,7 +41,7 @@ struct TestAndSetTest : SingleDiskPersistenceTestUtils { TestAndSetTest() : persistenceHandler(), asyncHandler(nullptr), - context(spi::LoadType(0, "default"), 0, 0) + context(0, 0) {} void SetUp() override { diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index bc52d7508dc..a1293a842c1 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -10,6 +10,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/error.h> #include <vespa/storage/common/bucket_resolver.h> #include <vespa/storage/storageserver/documentapiconverter.h> #include <vespa/storageapi/message/datagram.h> diff --git a/storage/src/tests/visiting/memory_bounded_trace_test.cpp b/storage/src/tests/visiting/memory_bounded_trace_test.cpp index 543531ef55a..e43fa702fb5 100644 --- a/storage/src/tests/visiting/memory_bounded_trace_test.cpp +++ b/storage/src/tests/visiting/memory_bounded_trace_test.cpp @@ -32,12 +32,12 @@ TEST(MemoryBoundedTraceTest, memory_used_is_accumulated_recursively_for_non_leaf TEST(MemoryBoundedTraceTest, trace_nodes_can_be_moved_and_implicitly_cleared) { MemoryBoundedTrace trace(100); EXPECT_TRUE(trace.add(mbus::TraceNode("hello world", epoch))); - mbus::TraceNode target; + mbus::Trace target; trace.moveTraceTo(target); EXPECT_EQ(1, target.getNumChildren()); EXPECT_EQ(0, trace.getApproxMemoryUsed()); - mbus::TraceNode emptinessCheck; + mbus::Trace emptinessCheck; trace.moveTraceTo(emptinessCheck); EXPECT_EQ(0, emptinessCheck.getNumChildren()); } @@ -52,7 +52,7 @@ TEST(MemoryBoundedTraceTest, trace_nodes_can_be_moved_and_implicitly_cleared) { TEST(MemoryBoundedTraceTest, moved_trace_tree_is_marked_as_strict) { MemoryBoundedTrace trace(100); EXPECT_TRUE(trace.add(mbus::TraceNode("hello world", epoch))); - mbus::TraceNode target; + mbus::Trace target; trace.moveTraceTo(target); EXPECT_EQ(1, target.getNumChildren()); EXPECT_TRUE(target.getChild(0).isStrict()); @@ -69,7 +69,7 @@ TEST(MemoryBoundedTraceTest, can_not_add_more_nodes_when_memory_used_exceeds_upp "the freeway", epoch))); EXPECT_EQ(11, trace.getApproxMemoryUsed()); - mbus::TraceNode target; + mbus::Trace target; trace.moveTraceTo(target); // Twice nested node (root -> added trace tree -> leaf with txt). EXPECT_EQ(1, target.getNumChildren()); @@ -82,7 +82,7 @@ TEST(MemoryBoundedTraceTest, moved_tree_includes_stats_node_when_nodes_omitted) EXPECT_TRUE(trace.add(mbus::TraceNode("abcdef", epoch))); EXPECT_FALSE(trace.add(mbus::TraceNode("ghijkjlmn", epoch))); - mbus::TraceNode target; + mbus::Trace target; trace.moveTraceTo(target); EXPECT_EQ(1, target.getNumChildren()); EXPECT_EQ(2, target.getChild(0).getNumChildren()); diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index ed54565c3c7..7943790f13d 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -3,9 +3,9 @@ #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 72668564345..f727cdf8eb2 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -799,15 +799,16 @@ TEST_F(VisitorTest, no_mbus_tracing_if_trace_level_is_zero) { cmd->getTrace().setLevel(0); std::shared_ptr<api::CreateVisitorReply> reply; ASSERT_NO_FATAL_FAILURE(doCompleteVisitingSession(cmd, reply)); - EXPECT_TRUE(reply->getTrace().getRoot().isEmpty()); + EXPECT_TRUE(reply->getTrace().isEmpty()); } TEST_F(VisitorTest, reply_contains_trace_if_trace_level_above_zero) { std::shared_ptr<api::CreateVisitorCommand> cmd(makeCreateVisitor()); cmd->getTrace().setLevel(1); + cmd->getTrace().trace(1,"at least one trace."); std::shared_ptr<api::CreateVisitorReply> reply; ASSERT_NO_FATAL_FAILURE(doCompleteVisitingSession(cmd, reply)); - EXPECT_FALSE(reply->getTrace().getRoot().isEmpty()); + EXPECT_FALSE(reply->getTrace().isEmpty()); } TEST_F(VisitorTest, no_more_iterators_sent_while_memory_used_above_limit) { diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index adb50465adc..e075f57514b 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -61,7 +61,7 @@ skip_thread bool default=false use_direct_storageapi_rpc bool default=false ## The number of network (FNET) threads used by the shared rpc resource. -rpc.num_network_threads int default=1 +rpc.num_network_threads int default=2 ## The number of (FNET) RPC targets to use per node in the cluster. ## diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index b756c5ca7bb..7f2a9b30a33 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -9,7 +9,6 @@ #include "operation_routing_snapshot.h" #include "outdated_nodes_map.h" #include <vespa/document/bucket/bucket.h> -#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/storagelink.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index c4fb9b8228c..c72e5731a59 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -144,9 +144,7 @@ GetOperation::onReceive(DistributorMessageSender& sender, const std::shared_ptr< LOG(debug, "Received %s", msg->toString(true).c_str()); - if ( ! getreply->getTrace().getRoot().isEmpty()) { - _msg->getTrace().getRoot().addChild(getreply->getTrace().getRoot()); - } + _msg->getTrace().addChild(getreply->steal_trace()); bool allDone = true; for (auto& response : _responses) { for (uint32_t i = 0; i < response.second.size(); i++) { diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.h b/storage/src/vespa/storage/distributor/operations/external/getoperation.h index 57e878d9e40..546cf7a7543 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.h @@ -6,6 +6,7 @@ #include <vespa/storage/distributor/operations/operation.h> #include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageframework/generic/clock/timer.h> #include <optional> diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.h b/storage/src/vespa/storage/distributor/operations/external/putoperation.h index 79b3dd74b82..61149839ed1 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/storage/distributor/operations/sequenced_operation.h> -#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storage/distributor/persistencemessagetracker.h> namespace document { diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 3866ee4e6f7..d7dd0be7f4f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -27,21 +27,22 @@ TwoPhaseUpdateOperation::TwoPhaseUpdateOperation( DistributorMetricSet& metrics, SequencingHandle sequencingHandle) : SequencedOperation(std::move(sequencingHandle)), - _updateMetric(metrics.updates[msg->getLoadType()]), - _putMetric(metrics.update_puts[msg->getLoadType()]), - _getMetric(metrics.update_gets[msg->getLoadType()]), - _metadata_get_metrics(metrics.update_metadata_gets[msg->getLoadType()]), - _updateCmd(std::move(msg)), - _updateReply(), - _manager(manager), - _bucketSpace(bucketSpace), - _sendState(SendState::NONE_SENT), - _mode(Mode::FAST_PATH), - _single_get_latency_timer(), - _fast_path_repair_source_node(0xffff), - _use_initial_cheap_metadata_fetch_phase( + _updateMetric(metrics.updates[msg->getLoadType()]), + _putMetric(metrics.update_puts[msg->getLoadType()]), + _getMetric(metrics.update_gets[msg->getLoadType()]), + _metadata_get_metrics(metrics.update_metadata_gets[msg->getLoadType()]), + _updateCmd(std::move(msg)), + _updateReply(), + _manager(manager), + _bucketSpace(bucketSpace), + _sendState(SendState::NONE_SENT), + _mode(Mode::FAST_PATH), + _trace(_updateCmd->getTrace().getLevel()), + _single_get_latency_timer(), + _fast_path_repair_source_node(0xffff), + _use_initial_cheap_metadata_fetch_phase( _manager.getDistributor().getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates()), - _replySent(false) + _replySent(false) { document::BucketIdFactory idFactory; _updateDocBucketId = idFactory.getBucketId(_updateCmd->getDocumentId()); @@ -132,9 +133,7 @@ TwoPhaseUpdateOperation::sendReply( std::shared_ptr<api::StorageReply>& reply) { assert(!_replySent); - if (!_trace.isEmpty()) { - reply->getTrace().getRoot().addChild(_trace); - } + reply->getTrace().addChild(std::move(_trace)); sender.sendReply(reply); _replySent = true; } @@ -645,11 +644,9 @@ TwoPhaseUpdateOperation::satisfiesUpdateTimestampConstraint(api::Timestamp ts) c } void -TwoPhaseUpdateOperation::addTraceFromReply(const api::StorageReply& reply) +TwoPhaseUpdateOperation::addTraceFromReply(api::StorageReply & reply) { - if ( ! reply.getTrace().getRoot().isEmpty()) { - _trace.addChild(reply.getTrace().getRoot()); - } + _trace.addChild(reply.steal_trace()); } void diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h index 2d8f3e8494d..a98fbe98c38 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h @@ -2,7 +2,6 @@ #pragma once #include "newest_replica.h" -#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storage/distributor/persistencemessagetracker.h> #include <vespa/storage/distributor/operations/sequenced_operation.h> #include <vespa/document/update/documentupdate.h> @@ -18,6 +17,7 @@ namespace storage { namespace api { class UpdateCommand; class CreateBucketReply; +class ReturnCode; } class UpdateMetricSet; @@ -125,7 +125,7 @@ private: DistributorMessageSender& sender, const document::Document& candidateDoc); bool satisfiesUpdateTimestampConstraint(api::Timestamp) const; - void addTraceFromReply(const api::StorageReply& reply); + void addTraceFromReply(api::StorageReply& reply); bool hasTasCondition() const noexcept; void replyWithTasFailure(DistributorMessageSender& sender, vespalib::stringref message); @@ -146,7 +146,7 @@ private: SentMessageMap _sentMessageMap; SendState _sendState; Mode _mode; - mbus::TraceNode _trace; + mbus::Trace _trace; document::BucketId _updateDocBucketId; std::vector<std::pair<document::BucketId, uint16_t>> _replicas_at_get_send_time; std::optional<framework::MilliSecTimer> _single_get_latency_timer; diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h index 9d9bce9160b..2e69a52a644 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storage/distributor/persistencemessagetracker.h> namespace document { diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp index 5a03e05d563..0868f18e88a 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp @@ -161,7 +161,7 @@ VisitorOperation::markOperationAsFailedDueToNodeError( result.getResult(), vespalib::make_string("[from content node %u] %s", fromFailingNodeIndex, - result.getMessage().c_str())); + vespalib::string(result.getMessage()).c_str())); } void @@ -171,7 +171,7 @@ VisitorOperation::onReceive( { api::CreateVisitorReply& reply = static_cast<api::CreateVisitorReply&>(*r); - _trace.add(reply.getTrace().getRoot()); + _trace.add(reply.steal_trace()); SentMessagesMap::iterator iter = _sentMessages.find(reply.getMsgId()); assert(iter != _sentMessages.end()); @@ -828,8 +828,8 @@ VisitorOperation::sendReply(const api::ReturnCode& code, DistributorMessageSende { if (!_sentReply) { // Send create visitor reply - api::CreateVisitorReply::SP reply(new api::CreateVisitorReply(*_msg)); - _trace.moveTraceTo(reply->getTrace().getRoot()); + auto reply = std::make_shared<api::CreateVisitorReply>(*_msg); + _trace.moveTraceTo(reply->getTrace()); reply->setLastBucket(getLastBucketVisited()); reply->setResult(code); diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h index 42b0bd56b9e..24c7157b481 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h @@ -38,7 +38,7 @@ public: const Config& config, VisitorMetricSet& metrics); - ~VisitorOperation(); + ~VisitorOperation() override; void onClose(DistributorMessageSender& sender) override; void onStart(DistributorMessageSender& sender) override; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp index 52c8344b820..bc77f4cf88f 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp @@ -105,7 +105,6 @@ IdealStateOperation::setCommandMeta(api::MaintenanceCommand& cmd) const { cmd.setPriority(_priority); cmd.setReason(_detailedReason); - cmd.setLoadType((*_manager->getLoadTypes())["maintenance"]); } std::string diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp index 9a94a5a62ad..3bd2406cd85 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp @@ -78,7 +78,7 @@ RemoveBucketOperation::onReceiveInternal(const std::shared_ptr<api::StorageReply "%s on node %u: %s. Reinserting node into bucket db with %s", getBucketId().toString().c_str(), node, - rep->getResult().getMessage().c_str(), + vespalib::string(rep->getResult().getMessage()).c_str(), rep->getBucketInfo().toString().c_str()); _manager->getDistributorComponent().updateBucketDatabase( @@ -104,8 +104,7 @@ RemoveBucketOperation::onReceiveInternal(const std::shared_ptr<api::StorageReply void -RemoveBucketOperation::onReceive(DistributorMessageSender&, - const std::shared_ptr<api::StorageReply> &msg) +RemoveBucketOperation::onReceive(DistributorMessageSender&, const std::shared_ptr<api::StorageReply> &msg) { if (onReceiveInternal(msg)) { done(); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index 57f8bc92316..508dcf13916 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -121,7 +121,7 @@ SplitOperation::onReceive(DistributorMessageSender&, const api::StorageReply::SP LOGBP(debug, "Split failed for %s: bucket not found. Storage and " "distributor bucket databases might be out of sync: %s", getBucketId().toString().c_str(), - rep.getResult().getMessage().c_str()); + vespalib::string(rep.getResult().getMessage()).c_str()); _ok = false; } else if (rep.getResult().isBusy()) { LOG(debug, "Split failed for %s, node was busy. Will retry later", diff --git a/storage/src/vespa/storage/distributor/operations/operation.cpp b/storage/src/vespa/storage/distributor/operations/operation.cpp index 9e9e3f8a60a..ee695dae606 100644 --- a/storage/src/vespa/storage/distributor/operations/operation.cpp +++ b/storage/src/vespa/storage/distributor/operations/operation.cpp @@ -2,7 +2,6 @@ #include "operation.h" #include <vespa/storage/common/distributorcomponent.h> -#include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/messageapi/storagecommand.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/vespalib/util/stringfmt.h> @@ -17,9 +16,7 @@ Operation::Operation() { } -Operation::~Operation() -{ -} +Operation::~Operation() = default; std::string Operation::getStatus() const @@ -42,7 +39,6 @@ Operation::copyMessageSettings(const api::StorageCommand& source, api::StorageCo target.getTrace().setLevel(source.getTrace().getLevel()); target.setTimeout(source.getTimeout()); target.setPriority(source.getPriority()); - target.setLoadType(source.getLoadType()); } } diff --git a/storage/src/vespa/storage/distributor/operations/operation.h b/storage/src/vespa/storage/distributor/operations/operation.h index 5f4dd63c2f6..538f15d2bf2 100644 --- a/storage/src/vespa/storage/distributor/operations/operation.h +++ b/storage/src/vespa/storage/distributor/operations/operation.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/vdslib/state/nodetype.h> #include <vespa/storage/distributor/distributormessagesender.h> #include <vespa/storageframework/generic/clock/time.h> diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index 275eba7f20c..2dba244dc9f 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -6,7 +6,6 @@ #include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/storageframework/generic/component/componentregister.h> #include <vespa/storageframework/generic/component/component.h> -#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/vespalib/stllike/hash_set.h> #include <boost/multi_index_container.hpp> diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp index 6362db6c002..584cb379400 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp @@ -22,6 +22,7 @@ PersistenceMessageTrackerImpl::PersistenceMessageTrackerImpl( _reply(std::move(reply)), _manager(link), _revertTimestamp(revertTimestamp), + _trace(_reply->getTrace().getLevel()), _requestTimer(link.getClock()), _n_persistence_replies_total(0), _n_successful_persistence_replies(0), @@ -163,25 +164,18 @@ PersistenceMessageTrackerImpl::addBucketInfoFromReply( bucket.toString().c_str(), bucketInfo.toString().c_str(), node); - _remapBucketInfo[bucket].push_back( - BucketCopy(_manager.getUniqueTimestamp(), - node, - bucketInfo)); + _remapBucketInfo[bucket].emplace_back(_manager.getUniqueTimestamp(), node, bucketInfo); } else { LOG(debug, "Bucket %s: Received bucket info %s from node %d", bucket.toString().c_str(), bucketInfo.toString().c_str(), node); - _bucketInfo[bucket].push_back( - BucketCopy(_manager.getUniqueTimestamp(), - node, - bucketInfo)); + _bucketInfo[bucket].emplace_back(_manager.getUniqueTimestamp(), node, bucketInfo); } } void -PersistenceMessageTrackerImpl::logSuccessfulReply(uint16_t node, - const api::BucketInfoReply& reply) const +PersistenceMessageTrackerImpl::logSuccessfulReply(uint16_t node, const api::BucketInfoReply& reply) const { LOG(spam, "Bucket %s: Received successful reply %s", reply.getBucketId().toString().c_str(), @@ -230,7 +224,7 @@ PersistenceMessageTrackerImpl::sendReply(MessageSender& sender) updateMetrics(); _trace.setStrict(false); if ( ! _trace.isEmpty()) { - _reply->getTrace().getRoot().addChild(_trace); + _reply->getTrace().addChild(std::move(_trace)); } sender.sendReply(_reply); @@ -292,9 +286,7 @@ PersistenceMessageTrackerImpl::updateFromReply( api::BucketInfoReply& reply, uint16_t node) { - if ( ! reply.getTrace().getRoot().isEmpty()) { - _trace.addChild(reply.getTrace().getRoot()); - } + _trace.addChild(reply.steal_trace()); if (reply.getType() == api::MessageType::CREATEBUCKET_REPLY) { handleCreateBucketReply(reply, node); diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.h b/storage/src/vespa/storage/distributor/persistencemessagetracker.h index 4392aa1fc30..cf9f4017eda 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.h +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.h @@ -73,7 +73,7 @@ private: DistributorComponent& _manager; api::Timestamp _revertTimestamp; std::vector<BucketNodePair> _revertNodes; - mbus::TraceNode _trace; + mbus::Trace _trace; framework::MilliSecTimer _requestTimer; uint32_t _n_persistence_replies_total; uint32_t _n_successful_persistence_replies; diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.cpp b/storage/src/vespa/storage/distributor/sentmessagemap.cpp index d74f52b2f86..6f1db429b97 100644 --- a/storage/src/vespa/storage/distributor/sentmessagemap.cpp +++ b/storage/src/vespa/storage/distributor/sentmessagemap.cpp @@ -3,6 +3,7 @@ #include "sentmessagemap.h" #include <vespa/storage/distributor/operations/operation.h> #include <sstream> +#include <set> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.map"); @@ -14,15 +15,13 @@ SentMessageMap::SentMessageMap() { } -SentMessageMap::~SentMessageMap() -{ -} +SentMessageMap::~SentMessageMap() = default; std::shared_ptr<Operation> SentMessageMap::pop() { - std::map<api::StorageMessage::Id, std::shared_ptr<Operation> >::iterator found = _map.begin(); + auto found = _map.begin(); if (found != _map.end()) { std::shared_ptr<Operation> retVal = found->second; @@ -36,11 +35,10 @@ SentMessageMap::pop() std::shared_ptr<Operation> SentMessageMap::pop(api::StorageMessage::Id id) { - std::map<api::StorageMessage::Id, std::shared_ptr<Operation> >::iterator found = _map.find(id); + auto found = _map.find(id); if (found != _map.end()) { - LOG(spam, "Found Id %" PRIu64 " in callback map: %p", id, - found->second.get()); + LOG(spam, "Found Id %" PRIu64 " in callback map: %p", id, found->second.get()); std::shared_ptr<Operation> retVal = found->second; _map.erase(found); @@ -55,8 +53,7 @@ SentMessageMap::pop(api::StorageMessage::Id id) void SentMessageMap::insert(api::StorageMessage::Id id, const std::shared_ptr<Operation> & callback) { - LOG(spam, "Inserting callback %p for message %" PRIu64 "", - callback.get(), id); + LOG(spam, "Inserting callback %p for message %" PRIu64 "", callback.get(), id); _map[id] = callback; } @@ -67,17 +64,11 @@ SentMessageMap::toString() const std::ostringstream ost; std::set<std::string> messages; - for (Map::const_iterator iter = _map.begin(); - iter != _map.end(); - ++iter) - { - messages.insert(iter->second.get()->toString()); + for (const auto & entry : _map) { + messages.insert(entry.second.get()->toString()); } - for (std::set<std::string>::const_iterator - it(messages.begin()), e(messages.end()); - it != e; ++it) - { - ost << *it << "\n"; + for (const auto & message : messages) { + ost << message << "\n"; } return ost.str(); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index aafc87aa84f..8b031af4b69 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -13,10 +13,10 @@ #pragma once -#include "mergestatus.h" #include <vespa/document/bucket/bucket.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/storage/common/messagesender.h> +#include <vespa/storageapi/messageapi/storagemessage.h> namespace storage { namespace api { @@ -34,6 +34,7 @@ struct FileStorMetrics; struct MessageSender; struct ServiceLayerComponentRegister; class AbortBucketOperationsCommand; +class MergeStatus; class FileStorHandler : public MessageSender { public: @@ -220,7 +221,7 @@ public: /** * Add a new merge state to the registry. */ - virtual void addMergeStatus(const document::Bucket&, MergeStatus::SP) = 0; + virtual void addMergeStatus(const document::Bucket&, std::shared_ptr<MergeStatus>) = 0; /** * Returns the reference to the current merge status for the given bucket. diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 01f44d2df42..5a7a598fb4c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -2,6 +2,7 @@ #include "filestorhandlerimpl.h" #include "filestormetrics.h" +#include "mergestatus.h" #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> @@ -77,7 +78,7 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripe FileStorHandlerImpl::~FileStorHandlerImpl() = default; void -FileStorHandlerImpl::addMergeStatus(const document::Bucket& bucket, MergeStatus::SP status) +FileStorHandlerImpl::addMergeStatus(const document::Bucket& bucket, std::shared_ptr<MergeStatus> status) { std::lock_guard mlock(_mergeStatesLock); if (_mergeStates.find(bucket) != _mergeStates.end()) { @@ -90,7 +91,7 @@ MergeStatus& FileStorHandlerImpl::editMergeStatus(const document::Bucket& bucket) { std::lock_guard mlock(_mergeStatesLock); - MergeStatus::SP status = _mergeStates[bucket]; + std::shared_ptr<MergeStatus> status = _mergeStates[bucket]; if ( ! status ) { throw vespalib::IllegalStateException("No merge state exist for " + bucket.toString(), VESPA_STRLOC); } @@ -140,7 +141,7 @@ FileStorHandlerImpl::clearMergeStatus(const document::Bucket& bucket, const api: return; } if (code != 0) { - MergeStatus::SP statusPtr(it->second); + std::shared_ptr<MergeStatus> statusPtr(it->second); assert(statusPtr.get()); MergeStatus& status(*statusPtr); if (status.reply.get()) { @@ -325,12 +326,9 @@ FileStorHandlerImpl::updateMetrics(const MetricLockGuard &) _metrics->pendingMerges.addValue(_mergeStates.size()); _metrics->queueSize.addValue(getQueueSize()); - for (auto & entry : _metrics->averageQueueWaitingTime.getMetricMap()) { - metrics::LoadType loadType(entry.first, "ignored"); - for (const auto & stripe : _metrics->stripes) { - const auto & m = stripe->averageQueueWaitingTime[loadType]; - entry.second->addTotalValueWithCount(m.getTotal(), m.getCount()); - } + for (const auto & stripe : _metrics->stripes) { + const auto & m = stripe->averageQueueWaitingTime; + _metrics->averageQueueWaitingTime.addTotalValueWithCount(m.getTotal(), m.getCount()); } } @@ -944,8 +942,7 @@ FileStorHandlerImpl::Stripe::get_next_async_message(monitor_guard& guard) FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx, PriorityIdx::iterator iter) { - api::StorageMessage & m(*iter->_command); - std::chrono::milliseconds waitTime(uint64_t(iter->_timer.stop(_metrics->averageQueueWaitingTime[m.getLoadType()]))); + std::chrono::milliseconds waitTime(uint64_t(iter->_timer.stop(_metrics->averageQueueWaitingTime))); std::shared_ptr<api::StorageMessage> msg = std::move(iter->_command); document::Bucket bucket(iter->_bucket); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 549de164229..b3d4ff0c730 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -16,11 +16,11 @@ #pragma once #include "filestorhandler.h" -#include "mergestatus.h" #include <vespa/document/bucket/bucketid.h> #include <vespa/metrics/metrics.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> +#include <vespa/storageapi/messageapi/storagereply.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> #include <boost/multi_index/member.hpp> @@ -218,7 +218,7 @@ public: return stripe(bucket).lock(bucket, lockReq); } - void addMergeStatus(const document::Bucket&, MergeStatus::SP) override; + void addMergeStatus(const document::Bucket&, std::shared_ptr<MergeStatus>) override; MergeStatus& editMergeStatus(const document::Bucket&) override; bool isMerging(const document::Bucket&) const override; uint32_t getNumActiveMerges() const override; @@ -242,7 +242,7 @@ private: MessageSender& _messageSender; const document::BucketIdFactory& _bucketIdFactory; mutable std::mutex _mergeStatesLock; - std::map<document::Bucket, MergeStatus::SP> _mergeStates; + std::map<document::Bucket, std::shared_ptr<MergeStatus>> _mergeStates; vespalib::duration _getNextMessageTimeout; const uint32_t _max_active_merges_per_stripe; // Read concurrently by stripes. mutable std::mutex _pauseMonitor; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index cba3969cd68..c1c412fefeb 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -673,10 +673,10 @@ FileStorManager::onInternal(const shared_ptr<api::InternalCommand>& msg) } case DestroyIteratorCommand::ID: { - spi::Context context(msg->getLoadType(), msg->getPriority(), msg->getTrace().getLevel()); + spi::Context context(msg->getPriority(), msg->getTrace().getLevel()); shared_ptr<DestroyIteratorCommand> cmd(std::static_pointer_cast<DestroyIteratorCommand>(msg)); _provider->destroyIterator(cmd->getIteratorId(), context); - msg->getTrace().getRoot().addChild(context.getTrace().getRoot()); + msg->getTrace().addChild(context.steal_trace()); return true; } case ReadBucketList::ID: diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 2953462dd1e..7abb41f4741 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -34,6 +34,7 @@ namespace storage { namespace api { class ReturnCode; class StorageReply; + class BucketCommand; } struct FileStorManagerTest; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index ed42f1932eb..996e3bfe515 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -189,27 +189,19 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: FileStorThreadMetrics::~FileStorThreadMetrics() = default; -FileStorStripeMetrics::FileStorStripeMetrics(const std::string& name, const std::string& description, - const LoadTypeSet& loadTypes) +FileStorStripeMetrics::FileStorStripeMetrics(const std::string& name, const std::string& description) : MetricSet(name, {{"partofsum"}}, description), - averageQueueWaitingTime(loadTypes, - metrics::DoubleAverageMetric("averagequeuewait", {}, - "Average time an operation spends in input queue."), - this) + averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this) { } FileStorStripeMetrics::~FileStorStripeMetrics() = default; -FileStorDiskMetrics::FileStorDiskMetrics(const std::string& name, const std::string& description, - const metrics::LoadTypeSet& loadTypes, MetricSet* owner) +FileStorDiskMetrics::FileStorDiskMetrics(const std::string& name, const std::string& description, MetricSet* owner) : MetricSet(name, {{"partofsum"}}, description, owner), sumThreads("allthreads", {{"sum"}}, "", this), sumStripes("allstripes", {{"sum"}}, "", this), - averageQueueWaitingTime(loadTypes, - metrics::DoubleAverageMetric("averagequeuewait", {}, - "Average time an operation spends in input queue."), - this), + averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this), queueSize("queuesize", {}, "Size of input message queue.", this), pendingMerges("pendingmerge", {}, "Number of buckets currently being merged.", this), waitingForLockHitRate("waitingforlockrate", {}, @@ -244,7 +236,7 @@ FileStorDiskMetrics::initDiskMetrics(const LoadTypeSet& loadTypes, uint32_t numS std::ostringstream name; name << "stripe" << i; desc << "Stripe " << i << '/' << numStripes; - stripes[i] = std::make_shared<FileStorStripeMetrics>(name.str(), desc.str(), loadTypes); + stripes[i] = std::make_shared<FileStorStripeMetrics>(name.str(), desc.str()); registerMetric(*stripes[i]); sumStripes.addMetricToSum(*stripes[i]); } @@ -267,7 +259,7 @@ void FileStorMetrics::initDiskMetrics(const LoadTypeSet& loadTypes, uint32_t num assert( ! disk); // Currently FileStorHandlerImpl expects metrics to exist for // disks that are not in use too. - disk = std::make_shared<FileStorDiskMetrics>( "disk_0", "Disk 0", loadTypes, this); + disk = std::make_shared<FileStorDiskMetrics>( "disk_0", "Disk 0", this); sum.addMetricToSum(*disk); disk->initDiskMetrics(loadTypes, numStripes, threadsPerDisk); } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index 86daf022fb2..aecbfc1ae2b 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -129,9 +129,8 @@ class FileStorStripeMetrics : public metrics::MetricSet { public: using SP = std::shared_ptr<FileStorStripeMetrics>; - metrics::LoadMetric<metrics::DoubleAverageMetric> averageQueueWaitingTime; - FileStorStripeMetrics(const std::string& name, const std::string& description, - const metrics::LoadTypeSet& loadTypes); + metrics::DoubleAverageMetric averageQueueWaitingTime; + FileStorStripeMetrics(const std::string& name, const std::string& description); ~FileStorStripeMetrics() override; }; @@ -144,14 +143,13 @@ public: std::vector<FileStorStripeMetrics::SP> stripes; metrics::SumMetric<MetricSet> sumThreads; metrics::SumMetric<MetricSet> sumStripes; - metrics::LoadMetric<metrics::DoubleAverageMetric> averageQueueWaitingTime; + metrics::DoubleAverageMetric averageQueueWaitingTime; metrics::LongAverageMetric queueSize; metrics::LongAverageMetric pendingMerges; metrics::DoubleAverageMetric waitingForLockHitRate; metrics::DoubleAverageMetric lockWaitTime; - FileStorDiskMetrics(const std::string& name, const std::string& description, - const metrics::LoadTypeSet& loadTypes, MetricSet* owner); + FileStorDiskMetrics(const std::string& name, const std::string& description, MetricSet* owner); ~FileStorDiskMetrics() override; void initDiskMetrics(const metrics::LoadTypeSet& loadTypes, uint32_t numStripes, uint32_t threadsPerDisk); diff --git a/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp b/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp index 2e390db69be..d75f09dfb3c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp @@ -9,12 +9,12 @@ LOG_SETUP(".mergestatus"); namespace storage { -MergeStatus::MergeStatus(const framework::Clock& clock, const metrics::LoadType& lt, +MergeStatus::MergeStatus(const framework::Clock& clock, api::StorageMessage::Priority priority, uint32_t traceLevel) : reply(), full_node_list(), nodeList(), maxTimestamp(0), diff(), pendingId(0), pendingGetDiff(), pendingApplyDiff(), timeout(0), startTime(clock), - context(lt, priority, traceLevel) + context(priority, traceLevel) {} MergeStatus::~MergeStatus() = default; @@ -30,8 +30,7 @@ MergeStatus::removeFromDiff( const std::vector<api::MergeBucketCommand::Node> &nodes) { std::deque<api::GetBucketDiffCommand::Entry>::iterator it(diff.begin()); - std::vector<api::ApplyBucketDiffCommand::Entry>::const_iterator it2( - part.begin()); + std::vector<api::ApplyBucketDiffCommand::Entry>::const_iterator it2(part.begin()); bool altered = false; HasMaskRemapper remap_mask(nodeList, nodes); // We expect part array to be sorted in the same order as in the diff, diff --git a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h index 51930f337c6..97a66ef56c1 100644 --- a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h +++ b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h @@ -15,8 +15,6 @@ namespace storage { class MergeStatus : public document::Printable { public: - using SP = std::shared_ptr<MergeStatus>; - std::shared_ptr<api::StorageReply> reply; std::vector<api::MergeBucketCommand::Node> full_node_list; std::vector<api::MergeBucketCommand::Node> nodeList; @@ -29,8 +27,8 @@ public: framework::MilliSecTimer startTime; spi::Context context; - MergeStatus(const framework::Clock&, const metrics::LoadType&, api::StorageMessage::Priority, uint32_t traceLevel); - ~MergeStatus(); + MergeStatus(const framework::Clock&, api::StorageMessage::Priority, uint32_t traceLevel); + ~MergeStatus() override; /** * Note: hasMask parameter and _entry._hasMask in part vector are per-reply masks, @@ -41,7 +39,7 @@ public: */ bool removeFromDiff(const std::vector<api::ApplyBucketDiffCommand::Entry>& part, uint16_t hasMask, const std::vector<api::MergeBucketCommand::Node> &nodes); void print(std::ostream& out, bool verbose, const std::string& indent) const override; - bool isFirstNode() const { return (reply.get() != 0); } + bool isFirstNode() const { return static_cast<bool>(reply); } }; } // storage diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index cab35e77bac..2e65302ee3b 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -4,6 +4,7 @@ #include "persistenceutil.h" #include "apply_bucket_diff_entry_complete.h" #include "apply_bucket_diff_entry_result.h" +#include <vespa/storage/persistence/filestorage/mergestatus.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/fieldset/fieldsets.h> @@ -145,7 +146,6 @@ MergeHandler::populateMetaData( bool MergeHandler::buildBucketInfoList( const spi::Bucket& bucket, - const documentapi::LoadType& /*loadType*/, Timestamp maxTimestamp, uint8_t myNodeIndex, std::vector<api::GetBucketDiffCommand::Entry>& output, @@ -320,7 +320,6 @@ namespace { void MergeHandler::fetchLocalData( const spi::Bucket& bucket, - const documentapi::LoadType& /*loadType*/, std::vector<api::ApplyBucketDiffCommand::Entry>& diff, uint8_t nodeIndex, spi::Context& context) const @@ -510,7 +509,6 @@ MergeHandler::applyDiffEntry(const spi::Bucket& bucket, api::BucketInfo MergeHandler::applyDiffLocally( const spi::Bucket& bucket, - const documentapi::LoadType& /*loadType*/, std::vector<api::ApplyBucketDiffCommand::Entry>& diff, uint8_t nodeIndex, spi::Context& context) const @@ -804,7 +802,7 @@ MergeHandler::processBucketMerge(const spi::Bucket& bucket, MergeStatus& status, cmd->setTimeout(status.timeout); if (applyDiffNeedLocalData(cmd->getDiff(), 0, true)) { framework::MilliSecTimer startTime(_clock); - fetchLocalData(bucket, cmd->getLoadType(), cmd->getDiff(), 0, context); + fetchLocalData(bucket, cmd->getDiff(), 0, context); _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); } status.pendingId = cmd->getMsgId(); @@ -847,7 +845,7 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP if (cmd.getNodes().size() < 2) { LOG(debug, "Attempt to merge a single instance of a bucket"); - tracker->fail(ReturnCode::ILLEGAL_PARAMETERS, "Cannot merge a single copy"); + tracker->fail(api::ReturnCode::ILLEGAL_PARAMETERS, "Cannot merge a single copy"); return tracker; } @@ -856,7 +854,7 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP for (uint16_t i = 0; i < cmd.getNodes().size(); ++i) { if (i == 0) { if (cmd.getNodes()[i].sourceOnly) { - tracker->fail(ReturnCode::ILLEGAL_PARAMETERS, + tracker->fail(api::ReturnCode::ILLEGAL_PARAMETERS, "Attempted to merge a chain where the first node " "in the chain is source only."); return tracker; @@ -865,7 +863,7 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP if (!cmd.getNodes()[i].sourceOnly && cmd.getNodes()[i-1].sourceOnly) { - tracker->fail(ReturnCode::ILLEGAL_PARAMETERS, + tracker->fail(api::ReturnCode::ILLEGAL_PARAMETERS, "Attempted to merge a chain where the source only " "copies are not in end of chain."); return tracker; @@ -876,15 +874,13 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP if (_env._fileStorHandler.isMerging(bucket.getBucket())) { const char* err = "A merge is already running on this bucket."; LOG(debug, "%s", err); - tracker->fail(ReturnCode::BUSY, err); + tracker->fail(api::ReturnCode::BUSY, err); return tracker; } checkResult(_spi.createBucket(bucket, tracker->context()), bucket, "create bucket"); MergeStateDeleter stateGuard(_env._fileStorHandler, bucket.getBucket()); - auto s = std::make_shared<MergeStatus>( - _clock, cmd.getLoadType(), - cmd.getPriority(), cmd.getTrace().getLevel()); + auto s = std::make_shared<MergeStatus>(_clock, cmd.getPriority(), cmd.getTrace().getLevel()); _env._fileStorHandler.addMergeStatus(bucket.getBucket(), s); s->full_node_list = cmd.getNodes(); s->nodeList = cmd.getNodes(); @@ -893,9 +889,9 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP s->startTime = framework::MilliSecTimer(_clock); auto cmd2 = std::make_shared<api::GetBucketDiffCommand>(bucket.getBucket(), s->nodeList, s->maxTimestamp.getTime()); - if (!buildBucketInfoList(bucket, cmd.getLoadType(), s->maxTimestamp, 0, cmd2->getDiff(), tracker->context())) { + if (!buildBucketInfoList(bucket, s->maxTimestamp, 0, cmd2->getDiff(), tracker->context())) { LOG(debug, "Bucket non-existing in db. Failing merge."); - tracker->fail(ReturnCode::BUCKET_DELETED, "Bucket not found in buildBucketInfo step"); + tracker->fail(api::ReturnCode::BUCKET_DELETED, "Bucket not found in buildBucketInfo step"); return tracker; } _env._metrics.merge_handler_metrics.mergeMetadataReadLatency.addValue(s->startTime.getElapsedTimeAsDouble()); @@ -1057,7 +1053,7 @@ MergeHandler::handleGetBucketDiff(api::GetBucketDiffCommand& cmd, MessageTracker checkResult(_spi.createBucket(bucket, tracker->context()), bucket, "create bucket"); if (_env._fileStorHandler.isMerging(bucket.getBucket())) { - tracker->fail(ReturnCode::BUSY, "A merge is already running on this bucket."); + tracker->fail(api::ReturnCode::BUSY, "A merge is already running on this bucket."); return tracker; } uint8_t index = findOwnIndex(cmd.getNodes(), _env._nodeIndex); @@ -1065,12 +1061,11 @@ MergeHandler::handleGetBucketDiff(api::GetBucketDiffCommand& cmd, MessageTracker std::vector<api::GetBucketDiffCommand::Entry>& remote(cmd.getDiff()); std::vector<api::GetBucketDiffCommand::Entry> local; framework::MilliSecTimer startTime(_clock); - if (!buildBucketInfoList(bucket, cmd.getLoadType(), - Timestamp(cmd.getMaxTimestamp()), + if (!buildBucketInfoList(bucket, Timestamp(cmd.getMaxTimestamp()), index, local, tracker->context())) { LOG(debug, "Bucket non-existing in db. Failing merge."); - tracker->fail(ReturnCode::BUCKET_DELETED, "Bucket not found in buildBucketInfo step"); + tracker->fail(api::ReturnCode::BUCKET_DELETED, "Bucket not found in buildBucketInfo step"); return tracker; } if (!mergeLists(remote, local, local)) { @@ -1106,9 +1101,7 @@ MergeHandler::handleGetBucketDiff(api::GetBucketDiffCommand& cmd, MessageTracker // When not the last node in merge chain, we must save reply, and // send command on. MergeStateDeleter stateGuard(_env._fileStorHandler, bucket.getBucket()); - auto s = std::make_shared<MergeStatus>(_clock, - cmd.getLoadType(), cmd.getPriority(), - cmd.getTrace().getLevel()); + auto s = std::make_shared<MergeStatus>(_clock, cmd.getPriority(), cmd.getTrace().getLevel()); _env._fileStorHandler.addMergeStatus(bucket.getBucket(), s); s->pendingGetDiff = std::make_shared<api::GetBucketDiffReply>(cmd); @@ -1241,8 +1234,7 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra LOG(debug, "%s", cmd.toString().c_str()); if (_env._fileStorHandler.isMerging(bucket.getBucket())) { - tracker->fail(ReturnCode::BUSY, - "A merge is already running on this bucket."); + tracker->fail(api::ReturnCode::BUSY, "A merge is already running on this bucket."); return tracker; } @@ -1250,19 +1242,17 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra bool lastInChain = index + 1u >= cmd.getNodes().size(); if (applyDiffNeedLocalData(cmd.getDiff(), index, !lastInChain)) { framework::MilliSecTimer startTime(_clock); - fetchLocalData(bucket, cmd.getLoadType(), cmd.getDiff(), index, tracker->context()); + fetchLocalData(bucket, cmd.getDiff(), index, tracker->context()); _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); } else { LOG(spam, "Merge(%s): Moving %zu entries, didn't need " "local data on node %u (%u).", - bucket.toString().c_str(), - cmd.getDiff().size(), - _env._nodeIndex, - index); + bucket.toString().c_str(), cmd.getDiff().size(), + _env._nodeIndex, index); } if (applyDiffHasLocallyNeededData(cmd.getDiff(), index)) { framework::MilliSecTimer startTime(_clock); - (void) applyDiffLocally(bucket, cmd.getLoadType(), cmd.getDiff(), index, tracker->context()); + (void) applyDiffLocally(bucket, cmd.getDiff(), index, tracker->context()); _env._metrics.merge_handler_metrics.mergeDataWriteLatency.addValue( startTime.getElapsedTimeAsDouble()); } else { @@ -1296,9 +1286,7 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra // When not the last node in merge chain, we must save reply, and // send command on. MergeStateDeleter stateGuard(_env._fileStorHandler, bucket.getBucket()); - auto s = std::make_shared<MergeStatus>(_clock, - cmd.getLoadType(), cmd.getPriority(), - cmd.getTrace().getLevel()); + auto s = std::make_shared<MergeStatus>(_clock, cmd.getPriority(), cmd.getTrace().getLevel()); _env._fileStorHandler.addMergeStatus(bucket.getBucket(), s); s->pendingApplyDiff = std::make_shared<api::ApplyBucketDiffReply>(cmd); @@ -1352,12 +1340,12 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag uint8_t index = findOwnIndex(reply.getNodes(), _env._nodeIndex); if (applyDiffNeedLocalData(diff, index, false)) { framework::MilliSecTimer startTime(_clock); - fetchLocalData(bucket, reply.getLoadType(), diff, index, s.context); + fetchLocalData(bucket, diff, index, s.context); _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); } if (applyDiffHasLocallyNeededData(diff, index)) { framework::MilliSecTimer startTime(_clock); - (void) applyDiffLocally(bucket, reply.getLoadType(), diff, index, s.context); + (void) applyDiffLocally(bucket, diff, index, s.context); _env._metrics.merge_handler_metrics.mergeDataWriteLatency.addValue(startTime.getElapsedTimeAsDouble()); } else { LOG(spam, "Merge(%s): Didn't need fetched data on node %u (%u)", diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index 64b1448577a..25b7f281ef0 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -16,15 +16,18 @@ #include "types.h" #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/docentry.h> -#include <vespa/storage/persistence/filestorage/mergestatus.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/common/messagesender.h> namespace storage { -namespace spi { struct PersistenceProvider; } +namespace spi { + struct PersistenceProvider; + class Context; +} class PersistenceUtil; class ApplyBucketDiffEntryResult; +class MergeStatus; class MergeHandler : public Types { @@ -42,19 +45,16 @@ public: bool buildBucketInfoList( const spi::Bucket& bucket, - const documentapi::LoadType&, Timestamp maxTimestamp, uint8_t myNodeIndex, std::vector<api::GetBucketDiffCommand::Entry>& output, spi::Context& context) const; void fetchLocalData(const spi::Bucket& bucket, - const documentapi::LoadType&, std::vector<api::ApplyBucketDiffCommand::Entry>& diff, uint8_t nodeIndex, spi::Context& context) const; api::BucketInfo applyDiffLocally( const spi::Bucket& bucket, - const documentapi::LoadType&, std::vector<api::ApplyBucketDiffCommand::Entry>& diff, uint8_t nodeIndex, spi::Context& context) const; diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 42d67573763..6e0004af88a 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "persistenceutil.h" +#include <vespa/storageapi/messageapi/bucketinforeply.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/bufferedlogger.h> @@ -42,7 +43,7 @@ MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, _updateBucketInfo(updateBucketInfo && hasBucketInfo(msg->getType().getId())), _bucketLock(std::move(bucketLock)), _msg(std::move(msg)), - _context(_msg->getLoadType(), _msg->getPriority(), _msg->getTrace().getLevel()), + _context(_msg->getPriority(), _msg->getTrace().getLevel()), _env(env), _replySender(replySender), _metric(nullptr), @@ -93,9 +94,7 @@ MessageTracker::sendReply() { _msg->toString(true).c_str(), vespalib::to_s(duration)); } if (hasReply()) { - if ( ! _context.getTrace().getRoot().isEmpty()) { - getReply().getTrace().getRoot().addChild(_context.getTrace().getRoot()); - } + getReply().getTrace().addChild(_context.steal_trace()); if (_updateBucketInfo) { if (getReply().getResult().success()) { _env.setBucketInfo(*this, _bucketLock->getBucket()); @@ -104,13 +103,10 @@ MessageTracker::sendReply() { if (getReply().getResult().success()) { _metric->latency.addValue(_timer.getElapsedTimeAsDouble()); } - LOG(spam, "Sending reply up: %s %" PRIu64, - getReply().toString().c_str(), getReply().getMsgId()); + LOG(spam, "Sending reply up: %s %" PRIu64, getReply().toString().c_str(), getReply().getMsgId()); _replySender.sendReplyDirectly(std::move(_reply)); } else { - if ( ! _context.getTrace().getRoot().isEmpty()) { - _msg->getTrace().getRoot().addChild(_context.getTrace().getRoot()); - } + _msg->getTrace().addChild(_context.steal_trace()); } } @@ -128,7 +124,7 @@ MessageTracker::checkForError(const spi::Result& response) } void -MessageTracker::fail(const ReturnCode& result) +MessageTracker::fail(const api::ReturnCode& result) { _result = result; LOG(debug, "Failing operation with error: %s", _result.toString().c_str()); diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index 8eeea6dddd2..47233b0ded6 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -6,11 +6,18 @@ #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/persistence/filestorage/filestorhandler.h> #include <vespa/storage/persistence/filestorage/filestormetrics.h> +#include <vespa/storageframework/generic/clock/timer.h> +#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/storage/storageutil/utils.h> +namespace storage::api { + class StorageMessage; + class StorageReply; +} namespace storage { + class PersistenceUtil; class MessageTracker : protected Types { @@ -18,7 +25,7 @@ public: typedef std::unique_ptr<MessageTracker> UP; MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, - FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg); + FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg); ~MessageTracker(); @@ -29,17 +36,17 @@ public: * non-default reply. They should call this function as soon as they create * a reply, to ensure it is stored in case of failure after reply creation. */ - void setReply(api::StorageReply::SP reply) { + void setReply(std::shared_ptr<api::StorageReply> reply) { assert( ! _reply ); _reply = std::move(reply); } /** Utility function to be able to write a bit less in client. */ void fail(uint32_t result, const String& message = "") { - fail(ReturnCode((api::ReturnCode::Result)result, message)); + fail(api::ReturnCode((api::ReturnCode::Result)result, message)); } /** Set the request to fail with the given failure. */ - void fail(const ReturnCode&); + void fail(const api::ReturnCode&); /** Don't send reply for the command being processed. Used by multi chain * commands like merge. */ @@ -52,7 +59,7 @@ public: api::StorageReply & getReply() { return *_reply; } - api::StorageReply::SP && stealReplySP() && { + std::shared_ptr<api::StorageReply> && stealReplySP() && { return std::move(_reply); } @@ -71,23 +78,23 @@ public: static MessageTracker::UP createForTesting(const framework::MilliSecTimer & timer, PersistenceUtil & env, MessageSender & replySender, - FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg); + FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg); private: MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, bool updateBucketInfo, - FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg); + FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg); [[nodiscard]] bool count_result_as_failure() const noexcept; bool _sendReply; bool _updateBucketInfo; FileStorHandler::BucketLockInterface::SP _bucketLock; - api::StorageMessage::SP _msg; + std::shared_ptr<api::StorageMessage> _msg; spi::Context _context; const PersistenceUtil &_env; MessageSender &_replySender; FileStorThreadMetrics::Op *_metric; // needs a better and thread safe solution - api::StorageReply::SP _reply; + std::shared_ptr<api::StorageReply> _reply; api::ReturnCode _result; framework::MilliSecTimer _timer; }; diff --git a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp index 3d5a4efc1af..6ed928245db 100644 --- a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp +++ b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp @@ -4,6 +4,7 @@ #include "persistenceutil.h" #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/storage/common/bucketoperationlogger.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldset/fieldsetrepo.h> diff --git a/storage/src/vespa/storage/persistence/splitjoinhandler.cpp b/storage/src/vespa/storage/persistence/splitjoinhandler.cpp index 6049dabc2fa..c64a892d6fb 100644 --- a/storage/src/vespa/storage/persistence/splitjoinhandler.cpp +++ b/storage/src/vespa/storage/persistence/splitjoinhandler.cpp @@ -7,6 +7,7 @@ #include "messages.h" #include <vespa/storage/common/bucketmessages.h> #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.splitjoinhandler"); @@ -297,19 +298,19 @@ bool SplitJoinHandler::validateJoinCommand(const api::JoinBucketsCommand& cmd, MessageTracker& tracker) { if (cmd.getSourceBuckets().size() != 2) { - tracker.fail(ReturnCode::ILLEGAL_PARAMETERS, + tracker.fail(api::ReturnCode::ILLEGAL_PARAMETERS, "Join needs exactly two buckets to be joined together" + cmd.getBucketId().toString()); return false; } // Verify that source and target buckets look sane. for (uint32_t i = 0; i < cmd.getSourceBuckets().size(); i++) { if (cmd.getSourceBuckets()[i] == cmd.getBucketId()) { - tracker.fail(ReturnCode::ILLEGAL_PARAMETERS, + tracker.fail(api::ReturnCode::ILLEGAL_PARAMETERS, "Join had both source and target bucket " + cmd.getBucketId().toString()); return false; } if (!cmd.getBucketId().contains(cmd.getSourceBuckets()[i])) { - tracker.fail(ReturnCode::ILLEGAL_PARAMETERS, + tracker.fail(api::ReturnCode::ILLEGAL_PARAMETERS, "Source bucket " + cmd.getSourceBuckets()[i].toString() + " is not contained in target " + cmd.getBucketId().toString()); return false; diff --git a/storage/src/vespa/storage/persistence/types.h b/storage/src/vespa/storage/persistence/types.h index 6a73adc8b0c..b36155bde8f 100644 --- a/storage/src/vespa/storage/persistence/types.h +++ b/storage/src/vespa/storage/persistence/types.h @@ -7,7 +7,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storageapi/buckets/bucketinfo.h> -#include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/defs.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/storageframework/generic/clock/time.h> @@ -25,8 +24,6 @@ struct Types { typedef Timestamp RevertToken; typedef vespalib::string String; typedef api::BucketInfo BucketInfo; - typedef api::ReturnCode ReturnCode; - typedef StorBucketDatabase::WrappedEntry BucketDBEntry; using MessageTrackerUP = std::unique_ptr<MessageTracker>; static const framework::MicroSecTime MAX_TIMESTAMP; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index a35b9d1d59a..c296f215c8c 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -102,7 +102,7 @@ CommunicationManager::handleMessage(std::unique_ptr<mbus::Message> msg) return; } - cmd->setTrace(docMsgPtr->getTrace()); + cmd->setTrace(docMsgPtr->steal_trace()); cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::move(docMsgPtr))); enqueue_or_process(std::move(cmd)); @@ -114,7 +114,7 @@ CommunicationManager::handleMessage(std::unique_ptr<mbus::Message> msg) //TODO: Can it be moved ? std::shared_ptr<api::StorageCommand> cmd = storMsgPtr->getCommand(); cmd->setTimeout(storMsgPtr->getTimeRemaining()); - cmd->setTrace(storMsgPtr->getTrace()); + cmd->setTrace(storMsgPtr->steal_trace()); cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::move(storMsgPtr))); enqueue_or_process(std::move(cmd)); @@ -203,12 +203,12 @@ CommunicationManager::handleReply(std::unique_ptr<mbus::Reply> reply) _docApiConverter.toStorageAPI(static_cast<documentapi::DocumentReply&>(*reply), *originalCommand)); if (sar) { - sar->setTrace(reply->getTrace()); + sar->setTrace(reply->steal_trace()); receiveStorageReply(sar); } } else if (protocolName == mbusprot::StorageProtocol::NAME) { mbusprot::StorageReply* sr(static_cast<mbusprot::StorageReply*>(reply.get())); - sr->getReply()->setTrace(reply->getTrace()); + sr->getReply()->setTrace(reply->steal_trace()); receiveStorageReply(sr->getReply()); } else { LOGBM(warning, "Received unsupported reply type %d for protocol '%s'.", @@ -260,7 +260,7 @@ convert_to_rpc_compression_config(const vespa::config::content::core::StorCommun CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri) : StorageLink("Communication manager"), _component(compReg, "communicationmanager"), - _metrics(_component.getLoadTypes()->getMetricLoadTypes()), + _metrics(), _shared_rpc_resources(), // Created upon initial configuration _storage_api_rpc_service(), // (ditto) _cc_rpc_service(), // (ditto) @@ -466,11 +466,11 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg) } LOG(spam, "Done processing: %s", msg->toString().c_str()); - _metrics.messageProcessTime[msg->getLoadType()].addValue(startTime.getElapsedTimeAsDouble()); + _metrics.messageProcessTime.addValue(startTime.getElapsedTimeAsDouble()); } catch (std::exception& e) { LOGBP(error, "When running command %s, caught exception %s. Discarding message", msg->toString().c_str(), e.what()); - _metrics.exceptionMessageProcessTime[msg->getLoadType()].addValue(startTime.getElapsedTimeAsDouble()); + _metrics.exceptionMessageProcessTime.addValue(startTime.getElapsedTimeAsDouble()); } } @@ -583,8 +583,8 @@ CommunicationManager::sendCommand( cmd->setContext(mbus::Context(msg->getMsgId())); cmd->setRetryEnabled(false); cmd->setTimeRemaining(msg->getTimeout()); - cmd->setTrace(msg->getTrace()); - sendMessageBusMessage(msg, std::move(cmd), address.getRoute()); + cmd->setTrace(msg->steal_trace()); + sendMessageBusMessage(msg, std::move(cmd), address.to_mbus_route()); } break; } @@ -596,14 +596,14 @@ CommunicationManager::sendCommand( if (mbusMsg) { MBUS_TRACE(msg->getTrace(), 7, "Communication manager: Converted OK"); - mbusMsg->setTrace(msg->getTrace()); + mbusMsg->setTrace(msg->steal_trace()); mbusMsg->setRetryEnabled(false); { std::lock_guard lock(_messageBusSentLock); _messageBusSent[msg->getMsgId()] = msg; } - sendMessageBusMessage(msg, std::move(mbusMsg), address.getRoute()); + sendMessageBusMessage(msg, std::move(mbusMsg), address.to_mbus_route()); break; } else { LOGBM(warning, "This type of message can't be sent via messagebus"); @@ -660,7 +660,8 @@ CommunicationManager::sendDirectRPCReply( activate_reply.activateVersion(), activate_reply.actualVersion()); } else { request.addReturnInt(reply->getResult().getResult()); - request.addReturnString(reply->getResult().getMessage().c_str()); + vespalib::stringref m = reply->getResult().getMessage(); + request.addReturnString(m.data(), m.size()); if (reply->getType() == api::MessageType::GETNODESTATE_REPLY) { api::GetNodeStateReply& gns(static_cast<api::GetNodeStateReply&>(*reply)); @@ -690,13 +691,13 @@ CommunicationManager::sendMessageBusReply( if (reply->getResult().getResult() == api::ReturnCode::WRONG_DISTRIBUTION) { replyUP = std::make_unique<documentapi::WrongDistributionReply>(reply->getResult().getMessage()); replyUP->swapState(*context._docAPIMsg); - replyUP->setTrace(reply->getTrace()); + replyUP->setTrace(reply->steal_trace()); replyUP->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_WRONG_DISTRIBUTION, reply->getResult().getMessage())); } else { replyUP = context._docAPIMsg->createReply(); replyUP->swapState(*context._docAPIMsg); - replyUP->setTrace(reply->getTrace()); + replyUP->setTrace(reply->steal_trace()); replyUP->setMessage(std::move(context._docAPIMsg)); _docApiConverter.transferReplyState(*reply, *replyUP); } @@ -707,7 +708,7 @@ CommunicationManager::sendMessageBusReply( } replyUP->swapState(*context._storageProtocolMsg); - replyUP->setTrace(reply->getTrace()); + replyUP->setTrace(reply->steal_trace()); replyUP->setMessage(std::move(context._storageProtocolMsg)); } diff --git a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.cpp b/storage/src/vespa/storage/storageserver/communicationmanagermetrics.cpp index 5f2bed07f66..8df19ba6639 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanagermetrics.cpp @@ -1,23 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "communicationmanagermetrics.h" -#include <vespa/documentapi/loadtypes/loadtypeset.h> using namespace metrics; namespace storage { -CommunicationManagerMetrics::CommunicationManagerMetrics(const LoadTypeSet& loadTypes, MetricSet* owner) +CommunicationManagerMetrics::CommunicationManagerMetrics(MetricSet* owner) : MetricSet("communication", {}, "Metrics for the communication manager", owner), queueSize("messagequeue", {}, "Size of input message queue.", this), - messageProcessTime(loadTypes, - DoubleAverageMetric("messageprocesstime", {}, - "Time transport thread uses to process a single message"), - this), - exceptionMessageProcessTime(loadTypes, - DoubleAverageMetric("exceptionmessageprocesstime", {}, - "Time transport thread uses to process a single message " - "that fails with an exception thrown into communication manager"), - this), + messageProcessTime("messageprocesstime", {}, "Time transport thread uses to process a single message", this), + exceptionMessageProcessTime("exceptionmessageprocesstime", {}, + "Time transport thread uses to process a single message " + "that fails with an exception thrown into communication manager", this), failedDueToTooLittleMemory("toolittlememory", {}, "Number of messages failed due to too little memory available", this), convertToStorageAPIFailures("convertfailures", {}, "Number of messages that failed to get converted to storage API messages", this), diff --git a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h b/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h index 2a2b8bdb5c1..cc62b93e6d4 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h +++ b/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h @@ -14,15 +14,15 @@ namespace storage { struct CommunicationManagerMetrics : public metrics::MetricSet { metrics::LongAverageMetric queueSize; - metrics::LoadMetric<metrics::DoubleAverageMetric> messageProcessTime; - metrics::LoadMetric<metrics::DoubleAverageMetric> exceptionMessageProcessTime; + metrics::DoubleAverageMetric messageProcessTime; + metrics::DoubleAverageMetric exceptionMessageProcessTime; metrics::LongCountMetric failedDueToTooLittleMemory; metrics::LongCountMetric convertToStorageAPIFailures; metrics::LongCountMetric bucketSpaceMappingFailures; metrics::DoubleAverageMetric sendCommandLatency; metrics::DoubleAverageMetric sendReplyLatency; - CommunicationManagerMetrics(const metrics::LoadTypeSet& loadTypes, metrics::MetricSet* owner = 0); + CommunicationManagerMetrics(metrics::MetricSet* owner = nullptr); ~CommunicationManagerMetrics(); }; diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index f0c987ee333..b1c8c1e43a4 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -14,6 +14,7 @@ #include <vespa/storageapi/message/searchresult.h> #include <vespa/storageapi/message/stat.h> #include <vespa/storageapi/message/visitor.h> +#include <vespa/messagebus/error.h> #include <vespa/log/log.h> LOG_SETUP(".documentapiconverter"); @@ -67,8 +68,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) case DocumentProtocol::MESSAGE_GETDOCUMENT: { documentapi::GetDocumentMessage& from(static_cast<documentapi::GetDocumentMessage&>(fromMsg)); - auto to = std::make_unique<api::GetCommand>(bucketResolver()->bucketFromId(from.getDocumentId()), from.getDocumentId(), from.getFieldSet()); - toMsg.reset(to.release()); + toMsg = std::make_unique<api::GetCommand>(bucketResolver()->bucketFromId(from.getDocumentId()), from.getDocumentId(), from.getFieldSet()); break; } case DocumentProtocol::MESSAGE_CREATEVISITOR: @@ -130,8 +130,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) { documentapi::RemoveLocationMessage& from(static_cast<documentapi::RemoveLocationMessage&>(fromMsg)); document::Bucket bucket(bucketResolver()->bucketSpaceFromName(from.getBucketSpace()), document::BucketId(0)); - api::RemoveLocationCommand::UP to(new api::RemoveLocationCommand(from.getDocumentSelection(), bucket)); - toMsg.reset(to.release()); + toMsg = std::make_unique<api::RemoveLocationCommand>(from.getDocumentSelection(), bucket); break; } default: @@ -145,10 +144,9 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) : 1ms*INT_MAX; toMsg->setTimeout(cappedTimeout); toMsg->setPriority(_priConverter->toStoragePriority(fromMsg.getPriority())); - toMsg->setLoadType(fromMsg.getLoadType()); - LOG(spam, "Converted command %s, loadtype %d, mapped priority %d to %d", - toMsg->toString().c_str(), toMsg->getLoadType().getId(), + LOG(spam, "Converted command %s, mapped priority %d to %d", + toMsg->toString().c_str(), fromMsg.getPriority(), toMsg->getPriority()); } return toMsg; diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 5e8e6809422..2e3c65ef70c 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -5,6 +5,7 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/persistence/messages.h> #include <vespa/messagebus/message.h> +#include <vespa/messagebus/error.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> @@ -762,7 +763,7 @@ MergeThrottler::handleMessageUp( if (mergeReply.getResult().getResult() != api::ReturnCode::OK) { LOG(debug, "Merging failed for %s (%s)", mergeReply.toString().c_str(), - mergeReply.getResult().getMessage().c_str()); + vespalib::string(mergeReply.getResult().getMessage()).c_str()); } processMergeReply(msg, true, msgGuard); diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index cb1d0380bf1..c465315a5a6 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -208,7 +208,7 @@ void StorageApiRpcService::encode_rpc_v1_response(FRT_RPCRequest& request, api:: // TODO skip encoding header altogether if no relevant fields set? protobuf::ResponseHeader hdr; if (reply.getTrace().getLevel() > 0) { - hdr.set_trace_payload(reply.getTrace().getRoot().encode()); + hdr.set_trace_payload(reply.getTrace().encode()); } // TODO consistent naming... encode_header_into_rpc_params(hdr, *ret); @@ -289,7 +289,7 @@ void StorageApiRpcService::RequestDone(FRT_RPCRequest* raw_req) { assert(reply); if (!hdr.trace_payload().empty()) { - cmd.getTrace().getRoot().addChild(mbus::TraceNode::decode(hdr.trace_payload())); + cmd.getTrace().addChild(mbus::TraceNode::decode(hdr.trace_payload())); } if (cmd.getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { cmd.getTrace().trace(TraceLevel::SEND_RECEIVE, diff --git a/storage/src/vespa/storage/visiting/memory_bounded_trace.cpp b/storage/src/vespa/storage/visiting/memory_bounded_trace.cpp index 9f9eb2942c1..05bb026f238 100644 --- a/storage/src/vespa/storage/visiting/memory_bounded_trace.cpp +++ b/storage/src/vespa/storage/visiting/memory_bounded_trace.cpp @@ -6,7 +6,7 @@ namespace storage { MemoryBoundedTrace::MemoryBoundedTrace(size_t softMemoryUpperBound) - : _node(), + : _trace(), _currentMemoryUsed(0), _omittedNodes(0), _omittedBytes(0), @@ -14,53 +14,49 @@ MemoryBoundedTrace::MemoryBoundedTrace(size_t softMemoryUpperBound) { } -namespace { - -size_t -computeTraceTreeMemoryUsage(const mbus::TraceNode& node) +bool +MemoryBoundedTrace::add(const mbus::TraceNode& node) { - if (node.isLeaf()) { - return node.getNote().size(); - } - size_t childSum = 0; - const uint32_t childCount = node.getNumChildren(); - for (uint32_t i = 0; i < childCount; ++i) { - childSum += computeTraceTreeMemoryUsage(node.getChild(i)); + const size_t nodeFootprint = node.computeMemoryUsage(); + + if (_currentMemoryUsed >= _softMemoryUpperBound) { + ++_omittedNodes; + _omittedBytes += nodeFootprint; + return false; } - return childSum; + _trace.addChild(vespalib::TraceNode(node)); + _currentMemoryUsed += nodeFootprint; + return true; } -} // anon ns - bool -MemoryBoundedTrace::add(const mbus::TraceNode& node) +MemoryBoundedTrace::add(mbus::Trace && node) { - const size_t nodeFootprint = computeTraceTreeMemoryUsage(node); + const size_t nodeFootprint = node.computeMemoryUsage(); if (_currentMemoryUsed >= _softMemoryUpperBound) { ++_omittedNodes; _omittedBytes += nodeFootprint; return false; } - _node.addChild(node); + _trace.addChild(std::move(node)); _currentMemoryUsed += nodeFootprint; return true; } void -MemoryBoundedTrace::moveTraceTo(mbus::TraceNode& out) +MemoryBoundedTrace::moveTraceTo(mbus::Trace& out) { - if (_node.isEmpty()) { + if (_trace.isEmpty()) { return; } if (_omittedNodes > 0) { - _node.addChild(vespalib::make_string( + _trace.trace(0, vespalib::make_string( "Trace too large; omitted %zu subsequent trace trees " "containing a total of %zu bytes", - _omittedNodes, _omittedBytes)); + _omittedNodes, _omittedBytes), false); } - out.addChild(_node); // XXX rvalue support should be added to TraceNode. - _node.clear(); + out.addChild(std::move(_trace)); _currentMemoryUsed = 0; _omittedNodes = 0; _omittedBytes = 0; diff --git a/storage/src/vespa/storage/visiting/memory_bounded_trace.h b/storage/src/vespa/storage/visiting/memory_bounded_trace.h index 71ce1be51ac..2c75c809af6 100644 --- a/storage/src/vespa/storage/visiting/memory_bounded_trace.h +++ b/storage/src/vespa/storage/visiting/memory_bounded_trace.h @@ -23,6 +23,7 @@ public: * otherwise. */ bool add(const mbus::TraceNode& node); + bool add(mbus::Trace && trace); /** * Append current trace tree to the output trace node and clear internal @@ -33,14 +34,14 @@ public: * * If current trace is empty, no nodes are added to `out`. */ - void moveTraceTo(mbus::TraceNode& out); + void moveTraceTo(mbus::Trace& out); size_t getApproxMemoryUsed() const noexcept { return _currentMemoryUsed; } private: - mbus::TraceNode _node; + mbus::Trace _trace; size_t _currentMemoryUsed; size_t _omittedNodes; size_t _omittedBytes; diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 982bcd78f6f..16a9b26a754 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -6,6 +6,7 @@ #include <vespa/storageapi/message/datagram.h> #include <vespa/storage/persistence/messages.h> #include <vespa/documentapi/messagebus/messages/visitor.h> +#include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/document/select/node.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -146,9 +147,7 @@ Visitor::BucketIterationState::~BucketIterationState() { if (_iteratorId != 0) { // Making the assumption that this is effectively nothrow. - std::shared_ptr<DestroyIteratorCommand> cmd( - new DestroyIteratorCommand(_iteratorId)); - cmd->setLoadType(_visitor._initiatingCmd->getLoadType()); + auto cmd = std::make_shared<DestroyIteratorCommand>(_iteratorId); cmd->getTrace().setLevel(_visitor._traceLevel); cmd->setPriority(0); @@ -178,7 +177,7 @@ Visitor::VisitorTarget::VisitorTarget() { } -Visitor::VisitorTarget::~VisitorTarget() {} +Visitor::VisitorTarget::~VisitorTarget() = default; Visitor::Visitor(StorageComponent& component) : _component(component), @@ -223,10 +222,9 @@ Visitor::sendMessage(documentapi::DocumentMessage::UP cmd) { assert(cmd.get()); if (!isRunning()) return; - cmd->setRoute(_dataDestination->getRoute()); + cmd->setRoute(*_dataDestination); cmd->setPriority(_documentPriority); - cmd->setLoadType(_initiatingCmd->getLoadType()); framework::MicroSecTime time(_component.getClock().getTimeInMicros()); @@ -293,7 +291,7 @@ Visitor::sendInfoMessage(documentapi::VisitorInfoMessage::UP cmd) if (!isRunning()) return; if (_controlDestination->toString().length()) { - cmd->setRoute(_controlDestination->getRoute()); + cmd->setRoute(*_controlDestination); cmd->setPriority(_documentPriority); cmd->setTimeRemaining(std::chrono::milliseconds(_visitorInfoTimeout.getTime())); auto& msgMeta = _visitorTarget.insertMessage(std::move(cmd)); @@ -372,10 +370,9 @@ Visitor::sendReplyOnce() std::shared_ptr<api::StorageReply> reply(_initiatingCmd->makeReply()); _hitCounter->updateVisitorStatistics(_visitorStatistics); - static_cast<api::CreateVisitorReply*>(reply.get()) - ->setVisitorStatistics(_visitorStatistics); + static_cast<api::CreateVisitorReply*>(reply.get())->setVisitorStatistics(_visitorStatistics); if (shouldAddMbusTrace()) { - _trace.moveTraceTo(reply->getTrace().getRoot()); + _trace.moveTraceTo(reply->getTrace()); } reply->setResult(_result); LOG(debug, "Sending %s", reply->toString(true).c_str()); @@ -557,8 +554,8 @@ Visitor::start(api::VisitorId id, api::StorageMessage::Id cmdId, void Visitor::attach(std::shared_ptr<api::StorageCommand> initiatingCmd, - const api::StorageMessageAddress& controlAddress, - const api::StorageMessageAddress& dataAddress, + const mbus::Route& controlAddress, + const mbus::Route& dataAddress, framework::MilliSecTime timeout) { _priority = initiatingCmd->getPriority(); @@ -572,9 +569,8 @@ Visitor::attach(std::shared_ptr<api::StorageCommand> initiatingCmd, _traceLevel = _initiatingCmd->getTrace().getLevel(); { // Set new address - _controlDestination.reset( - new api::StorageMessageAddress(controlAddress)); - _dataDestination.reset(new api::StorageMessageAddress(dataAddress)); + _controlDestination = std::make_unique<mbus::Route>(controlAddress); + _dataDestination = std::make_unique<mbus::Route>(dataAddress); } LOG(debug, "Visitor '%s' has control destination %s and data " "destination %s.", @@ -603,15 +599,14 @@ bool Visitor::addBoundedTrace(uint32_t level, const vespalib::string &message) { mbus::Trace tempTrace; tempTrace.trace(level, message); - return _trace.add(tempTrace.getRoot()); + return _trace.add(std::move(tempTrace)); } void -Visitor::handleDocumentApiReply(mbus::Reply::UP reply, - VisitorThreadMetrics& metrics) +Visitor::handleDocumentApiReply(mbus::Reply::UP reply, VisitorThreadMetrics& metrics) { if (shouldAddMbusTrace()) { - _trace.add(reply->getTrace().getRoot()); + _trace.add(reply->steal_trace()); } mbus::Message::UP message = reply->getMessage(); @@ -736,7 +731,6 @@ Visitor::onCreateIteratorReply( LOG(debug, "Visitor '%s' starting to visit bucket %s.", _id.c_str(), bucketId.toString().c_str()); auto cmd = std::make_shared<GetIterCommand>(bucket, bucketState.getIteratorId(), _docBlockSize); - cmd->setLoadType(_initiatingCmd->getLoadType()); cmd->getTrace().setLevel(_traceLevel); cmd->setPriority(_priority); ++bucketState._pendingIterators; @@ -832,7 +826,7 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, } if (shouldAddMbusTrace()) { - _trace.add(reply->getTrace().getRoot()); + _trace.add(reply->steal_trace()); } LOG(debug, "Continuing visitor %s.", _id.c_str()); @@ -1142,7 +1136,6 @@ Visitor::getIterators() } auto cmd = std::make_shared<GetIterCommand>( bucketState.getBucket(), bucketState.getIteratorId(), _docBlockSize); - cmd->setLoadType(_initiatingCmd->getLoadType()); cmd->getTrace().setLevel(_traceLevel); cmd->setPriority(_priority); _messageHandler->send(cmd, *this); @@ -1188,7 +1181,6 @@ Visitor::getIterators() spi::NEWEST_DOCUMENT_OR_REMOVE : spi::NEWEST_DOCUMENT_ONLY)); - cmd->setLoadType(_initiatingCmd->getLoadType()); cmd->getTrace().setLevel(_traceLevel); cmd->setPriority(_initiatingCmd->getPriority()); cmd->setReadConsistency(getRequiredReadConsistency()); diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 8a1f675a4c5..8eb02e6ccfc 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -330,8 +330,8 @@ protected: documentapi::Priority::Value _documentPriority; std::string _id; - std::unique_ptr<api::StorageMessageAddress> _controlDestination; - std::unique_ptr<api::StorageMessageAddress> _dataDestination; + std::unique_ptr<mbus::Route> _controlDestination; + std::unique_ptr<mbus::Route> _dataDestination; std::shared_ptr<document::select::Node> _documentSelection; std::string _documentSelectionString; vdslib::VisitorStatistics _visitorStatistics; @@ -355,10 +355,12 @@ public: framework::MicroSecTime getStartTime() const { return _startTime; } api::VisitorId getVisitorId() const { return _visitorId; } const std::string& getVisitorName() const { return _id; } - const api::StorageMessageAddress* getControlDestination() const - { return _controlDestination.get(); } // Can't be null if attached - const api::StorageMessageAddress* getDataDestination() const - { return _dataDestination.get(); } // Can't be null if attached + const mbus::Route* getControlDestination() const { + return _controlDestination.get(); // Can't be null if attached + } + const mbus::Route* getDataDestination() const { + return _dataDestination.get(); // Can't be null if attached + } void setMaxPending(unsigned int maxPending) { _visitorOptions._maxPending = maxPending; } @@ -471,8 +473,8 @@ public: documentapi::Priority::Value); void attach(std::shared_ptr<api::StorageCommand> initiatingCmd, - const api::StorageMessageAddress& controlAddress, - const api::StorageMessageAddress& dataAddress, + const mbus::Route& controlAddress, + const mbus::Route& dataAddress, framework::MilliSecTime timeout); void handleDocumentApiReply(mbus::Reply::UP reply, diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index acaa96bc418..2839d3566aa 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -2,6 +2,7 @@ #include "visitorthread.h" #include "messages.h" +#include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/document/select/bodyfielddetector.h> #include <vespa/document/select/parser.h> #include <vespa/messagebus/rpcmessagebus.h> @@ -381,19 +382,18 @@ VisitorThread::createVisitor(vespalib::stringref libName, } namespace { - std::unique_ptr<api::StorageMessageAddress> - getDataAddress(const api::CreateVisitorCommand& cmd) - { - return std::make_unique<api::StorageMessageAddress>( - mbus::Route::parse(cmd.getDataDestination())); - } - std::unique_ptr<api::StorageMessageAddress> - getControlAddress(const api::CreateVisitorCommand& cmd) - { - return std::make_unique<api::StorageMessageAddress>( - mbus::Route::parse(cmd.getControlDestination())); - } +std::unique_ptr<mbus::Route> +getDataAddress(const api::CreateVisitorCommand& cmd) +{ + return std::make_unique<mbus::Route>(mbus::Route::parse(cmd.getDataDestination())); +} + +std::unique_ptr<mbus::Route> +getControlAddress(const api::CreateVisitorCommand& cmd) +{ + return std::make_unique<mbus::Route>(mbus::Route::parse(cmd.getControlDestination())); +} void validateDocumentSelection(const document::DocumentTypeRepo& repo, @@ -423,8 +423,8 @@ VisitorThread::onCreateVisitor( assert(_currentlyRunningVisitor == _visitors.end()); ReturnCode result(ReturnCode::OK); std::unique_ptr<document::select::Node> docSelection; - std::unique_ptr<api::StorageMessageAddress> controlAddress; - std::unique_ptr<api::StorageMessageAddress> dataAddress; + std::unique_ptr<mbus::Route> controlAddress; + std::unique_ptr<mbus::Route> dataAddress; std::shared_ptr<Visitor> visitor; do { // If no buckets are specified, fail command diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 9fd2c96f27e..e413a62ae39 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -16,9 +16,7 @@ #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/vespalib/util/growablebytebuffer.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <iomanip> #include <sstream> #include <vespa/vespalib/gtest/gtest.h> @@ -119,7 +117,7 @@ namespace { TEST_F(StorageProtocolTest, testAddress50) { StorageMessageAddress address("foo", lib::NodeType::STORAGE, 3); EXPECT_EQ(vespalib::string("storage/cluster.foo/storage/3/default"), - address.getRoute().toString()); + address.to_mbus_route().toString()); } template<typename Command> std::shared_ptr<Command> @@ -162,11 +160,9 @@ StorageProtocolTest::copyReply(const std::shared_ptr<Reply>& m) TEST_P(StorageProtocolTest, put) { auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); cmd->setUpdateTimestamp(Timestamp(13)); - cmd->setLoadType(_loadTypes["foo"]); auto cmd2 = copyCommand(cmd); EXPECT_EQ(_bucket, cmd2->getBucket()); EXPECT_EQ(*_testDoc, *cmd2->getDocument()); - EXPECT_EQ(vespalib::string("foo"), cmd2->getLoadType().getName()); EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); EXPECT_EQ(Timestamp(13), cmd2->getUpdateTimestamp()); @@ -221,12 +217,10 @@ TEST_P(StorageProtocolTest, request_metadata_is_propagated) { auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); cmd->forceMsgId(12345); cmd->setPriority(50); - cmd->setLoadType(_loadTypes["foo"]); cmd->setSourceIndex(321); auto cmd2 = copyCommand(cmd); EXPECT_EQ(12345, cmd2->getMsgId()); EXPECT_EQ(50, cmd2->getPriority()); - EXPECT_EQ(_loadTypes["foo"].getId(), cmd2->getLoadType().getId()); EXPECT_EQ(321, cmd2->getSourceIndex()); } @@ -821,4 +815,26 @@ TEST_P(StorageProtocolTest, serialized_size_is_used_to_set_approx_size_of_storag } } +TEST_P(StorageProtocolTest, track_memory_footprint_for_some_messages) { + EXPECT_EQ(64u, sizeof(StorageMessage)); + EXPECT_EQ(80u, sizeof(StorageReply)); + EXPECT_EQ(104u, sizeof(BucketReply)); + EXPECT_EQ(8u, sizeof(document::BucketId)); + EXPECT_EQ(16u, sizeof(document::Bucket)); + EXPECT_EQ(32u, sizeof(BucketInfo)); + EXPECT_EQ(136u, sizeof(BucketInfoReply)); + EXPECT_EQ(280u, sizeof(PutReply)); + EXPECT_EQ(264u, sizeof(UpdateReply)); + EXPECT_EQ(256u, sizeof(RemoveReply)); + EXPECT_EQ(344u, sizeof(GetReply)); + EXPECT_EQ(80u, sizeof(StorageCommand)); + EXPECT_EQ(104u, sizeof(BucketCommand)); + EXPECT_EQ(104u, sizeof(BucketInfoCommand)); + EXPECT_EQ(104u + sizeof(std::string), sizeof(TestAndSetCommand)); + EXPECT_EQ(136u + sizeof(std::string), sizeof(PutCommand)); + EXPECT_EQ(136u + sizeof(std::string), sizeof(UpdateCommand)); + EXPECT_EQ(216u + sizeof(std::string), sizeof(RemoveCommand)); + EXPECT_EQ(288u, sizeof(GetCommand)); +} + } // storage::api diff --git a/storageapi/src/tests/messageapi/storage_message_address_test.cpp b/storageapi/src/tests/messageapi/storage_message_address_test.cpp index c340cba4b28..f7f254e9119 100644 --- a/storageapi/src/tests/messageapi/storage_message_address_test.cpp +++ b/storageapi/src/tests/messageapi/storage_message_address_test.cpp @@ -22,15 +22,19 @@ TEST(StorageMessageAddressTest, storage_hash_covers_all_expected_fields) { hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 123), hash_of("foo", lib::NodeType::STORAGE, 123)); + EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("bar", lib::NodeType::STORAGE, 0)); // These tests are all true with extremely high probability, though they do // depend on a hash function that may inherently cause collisions. EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), - hash_of("bar", lib::NodeType::STORAGE, 0)); - EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), hash_of("foo", lib::NodeType::STORAGE, 1)); + + EXPECT_EQ(88u, sizeof(StorageMessageAddress)); + EXPECT_EQ(64u, sizeof(StorageMessage)); + EXPECT_EQ(16u, sizeof(mbus::Trace)); } } // storage::api diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index 8a0204c8fc9..2c7b4a1e6f8 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -102,7 +102,7 @@ ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, api::StorageCommand& msg) c uint8_t priority = SH::getByte(buf); msg.setPriority(priority); msg.setSourceIndex(SH::getShort(buf)); - msg.setLoadType(_loadTypes[SH::getInt(buf)]); + (void)SH::getInt(buf); } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp index b6d0d297a69..ee577881d82 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp @@ -5,8 +5,7 @@ #include "storagecommand.h" #include "serializationhelper.h" -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { using documentapi::TestAndSetCondition; @@ -62,5 +61,4 @@ void ProtocolSerialization5_2::encodeTasCondition(GBBuf & buf, const api::Storag buf.putString(cmd.getCondition().getSelection()); } -} // mbusprot -} // storage +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h index 42e9c17e192..f6f9443248c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h @@ -7,8 +7,7 @@ #include <vespa/vespalib/util/growablebytebuffer.h> #include <vespa/storageapi/message/persistence.h> -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { class ProtocolSerialization5_2 : public ProtocolSerialization5_1 { @@ -31,5 +30,4 @@ protected: static void encodeTasCondition(GBBuf & buf, const api::StorageCommand & cmd); }; -} // mbusprot -} // storage +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 5d66d86036c..b356ce59999 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -253,7 +253,6 @@ public: dest.forceMsgId(_hdr.message_id()); dest.setPriority(static_cast<uint8_t>(_hdr.priority())); dest.setSourceIndex(static_cast<uint16_t>(_hdr.source_index())); - dest.setLoadType(_load_types[_hdr.loadtype_id()]); } ProtobufType& request() noexcept { return *_proto_obj; } diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index e5ec7698329..072d02c7d32 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -199,7 +199,7 @@ public: GetBucketDiffCommand(const document::Bucket &bucket, const std::vector<Node>&, Timestamp maxTimestamp); - ~GetBucketDiffCommand(); + ~GetBucketDiffCommand() override; const std::vector<Node>& getNodes() const { return _nodes; } Timestamp getMaxTimestamp() const { return _maxTimestamp; } diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h index 67c41a0cc4d..8440591ecde 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ b/storageapi/src/vespa/storageapi/message/visitor.h @@ -205,18 +205,19 @@ public: VisitorInfoCommand(); ~VisitorInfoCommand() override; - void setErrorCode(const ReturnCode& code) { _error = code; } + void setErrorCode(ReturnCode && code) { _error = std::move(code); } void setCompleted() { _completed = true; } - void setBucketCompleted(const document::BucketId& id, Timestamp lastVisited) - { + void setBucketCompleted(const document::BucketId& id, Timestamp lastVisited) { _bucketsCompleted.push_back(BucketTimestampPair(id, lastVisited)); } - void setBucketsCompleted(const std::vector<BucketTimestampPair>& bc) - { _bucketsCompleted = bc; } + void setBucketsCompleted(const std::vector<BucketTimestampPair>& bc) { + _bucketsCompleted = bc; + } const ReturnCode& getErrorCode() const { return _error; } - const std::vector<BucketTimestampPair>& getCompletedBucketsList() const - { return _bucketsCompleted; } + const std::vector<BucketTimestampPair>& getCompletedBucketsList() const { + return _bucketsCompleted; + } bool visitorCompleted() const { return _completed; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp index 1385fc331ac..e5fb7764aeb 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp @@ -7,8 +7,7 @@ using document::Bucket; using document::BucketId; -namespace storage { -namespace api { +namespace storage::api { BucketReply::BucketReply(const BucketCommand& cmd, const ReturnCode& code) @@ -42,5 +41,4 @@ BucketReply::print(std::ostream& out, bool verbose, } } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp index d5c8cb7da68..dbe2f602703 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp @@ -10,17 +10,39 @@ ReturnCode::ReturnCode() _message() {} -ReturnCode::ReturnCode(const ReturnCode &) = default; -ReturnCode & ReturnCode::operator = (const ReturnCode &) = default; ReturnCode & ReturnCode::operator = (ReturnCode &&) noexcept = default; ReturnCode::~ReturnCode() = default; -ReturnCode::ReturnCode(Result result, vespalib::stringref msg) +ReturnCode::ReturnCode(Result result) : _result(result), - _message(msg) + _message() {} -vespalib::string ReturnCode::getResultString(Result result) { +ReturnCode::ReturnCode(Result result, vespalib::stringref msg) + : _result(result), + _message() +{ + if ( ! msg.empty()) { + _message = std::make_unique<vespalib::string>(msg); + } +} + +ReturnCode::ReturnCode(const ReturnCode & rhs) + : _result(rhs._result), + _message() +{ + if (rhs._message) { + _message = std::make_unique<vespalib::string>(*rhs._message); + } +} + +ReturnCode & +ReturnCode::operator = (const ReturnCode & rhs) { + return operator=(ReturnCode(rhs)); +} + +vespalib::string +ReturnCode::getResultString(Result result) { return documentapi::DocumentProtocol::getErrorName(result); } @@ -28,9 +50,9 @@ vespalib::string ReturnCode::toString() const { vespalib::string ret = "ReturnCode("; ret += getResultString(_result); - if ( ! _message.empty()) { + if ( _message && ! _message->empty()) { ret += ", "; - ret += _message; + ret += *_message; } ret += ")"; return ret; @@ -146,4 +168,20 @@ ReturnCode::isBucketDisappearance() const } } +vespalib::stringref +ReturnCode::getMessage() const { + return _message + ? _message->operator vespalib::stringref() + : vespalib::stringref(); +} + +bool +ReturnCode::operator==(const ReturnCode& code) const { + return (_result == code._result) && (getMessage() == code.getMessage()); +} + +bool +ReturnCode::operator!=(const ReturnCode& code) const { + return (_result != code._result) || (getMessage() != code.getMessage()); +} } diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storageapi/src/vespa/storageapi/messageapi/returncode.h index 0149ae29a05..bef59a334d9 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.h +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.h @@ -59,18 +59,18 @@ public: private: Result _result; - vespalib::string _message; + std::unique_ptr<vespalib::string> _message; public: ReturnCode(); - explicit ReturnCode(Result result, vespalib::stringref msg = ""); + explicit ReturnCode(Result result); + explicit ReturnCode(Result result, vespalib::stringref msg); ReturnCode(const ReturnCode &); ReturnCode & operator = (const ReturnCode &); ReturnCode(ReturnCode &&) noexcept = default; ReturnCode & operator = (ReturnCode &&) noexcept; ~ReturnCode(); - const vespalib::string& getMessage() const { return _message; } - void setMessage(vespalib::stringref message) { _message = message; } + vespalib::stringref getMessage() const; Result getResult() const { return _result; } @@ -85,10 +85,8 @@ public: bool operator==(Result res) const { return _result == res; } bool operator!=(Result res) const { return _result != res; } - bool operator==(const ReturnCode& code) const - { return _result == code._result && _message == code._message; } - bool operator!=(const ReturnCode& code) const - { return _result != code._result || _message != code._message; } + bool operator==(const ReturnCode& code) const; + bool operator!=(const ReturnCode& code) const; // To avoid lots of code matching various return codes in storage, we define // some functions they can use to match those codes that corresponds to what diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp b/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp index d9bbf34141a..397c869f4fb 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp @@ -11,7 +11,6 @@ StorageCommand::StorageCommand(const StorageCommand& other) _timeout(other._timeout), _sourceIndex(other._sourceIndex) { - setTrace(other.getTrace()); } StorageCommand::StorageCommand(const MessageType& type, Priority p) diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h b/storageapi/src/vespa/storageapi/messageapi/storagecommand.h index c835168c5b7..bd55ee57a8f 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagecommand.h @@ -30,7 +30,7 @@ protected: public: DECLARE_POINTER_TYPEDEFS(StorageCommand); - virtual ~StorageCommand(); + ~StorageCommand() override; bool sourceIndexSet() const { return (_sourceIndex != 0xffff); } void setSourceIndex(uint16_t sourceIndex) { _sourceIndex = sourceIndex; } diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 0f5546a5510..9c5df379d22 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "storagemessage.h" - +#include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/messagebus/routing/verbatimdirective.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -139,15 +139,6 @@ MessageType::print(std::ostream& out, bool verbose, const std::string& indent) c out << ")"; } -StorageMessageAddress::StorageMessageAddress(const mbus::Route& route) - : _route(route), - _protocol(DOCUMENT), - _precomputed_storage_hash(0), - _cluster(""), - _type(nullptr), - _index(0xFFFF) -{ } - std::ostream & operator << (std::ostream & os, const StorageMessageAddress & addr) { return os << addr.toString(); } @@ -160,39 +151,45 @@ createAddress(vespalib::stringref cluster, const lib::NodeType& type, uint16_t i return os.str(); } +size_t +calculate_node_hash(const lib::NodeType& type, uint16_t index) +{ + uint16_t buf[] = { type, index }; + return vespalib::hashValue(&buf, sizeof(buf)); +} + // TODO we ideally want this removed. Currently just in place to support usage as map key when emplacement not available StorageMessageAddress::StorageMessageAddress() - : _route(), - _protocol(Protocol::STORAGE), + : _cluster(), _precomputed_storage_hash(0), - _cluster(), _type(nullptr), + _protocol(Protocol::STORAGE), _index(0) {} - StorageMessageAddress::StorageMessageAddress(vespalib::stringref cluster, const lib::NodeType& type, uint16_t index, Protocol protocol) - : _route(), - _protocol(protocol), - _precomputed_storage_hash(0), - _cluster(cluster), + : _cluster(cluster), + _precomputed_storage_hash(calculate_node_hash(type, index)), _type(&type), + _protocol(protocol), _index(index) { - std::vector<mbus::IHopDirective::SP> directives; - auto address_as_str = createAddress(cluster, type, index); - // We reuse the string representation and pass it to vespalib's hashValue instead of - // explicitly combining a running hash over the individual fields. This is because - // hashValue internally uses xxhash, which offers great dispersion of bits even for - // minimal changes in the input (such as single bit differences in the index). - _precomputed_storage_hash = vespalib::hashValue(address_as_str.data(), address_as_str.size()); - directives.emplace_back(std::make_shared<mbus::VerbatimDirective>(std::move(address_as_str))); - _route.addHop(mbus::Hop(std::move(directives), false)); } StorageMessageAddress::~StorageMessageAddress() = default; +mbus::Route +StorageMessageAddress::to_mbus_route() const +{ + mbus::Route result; + auto address_as_str = createAddress(_cluster, *_type, _index); + std::vector<mbus::IHopDirective::SP> directives; + directives.emplace_back(std::make_shared<mbus::VerbatimDirective>(std::move(address_as_str))); + result.addHop(mbus::Hop(std::move(directives), false)); + return result; +} + uint16_t StorageMessageAddress::getIndex() const { @@ -251,7 +248,7 @@ StorageMessageAddress::print(vespalib::asciistream & out) const out << "Document protocol"; } if (!_type) { - out << ", " << _route.toString() << ")"; + out << ", " << to_mbus_route().toString() << ")"; } else { out << ", cluster " << _cluster << ", nodetype " << *_type << ", index " << _index << ")"; @@ -269,26 +266,32 @@ StorageMessage::generateMsgId() StorageMessage::StorageMessage(const MessageType& type, Id id) : _type(type), _msgId(id), - _priority(NORMAL), _address(), - _loadType(documentapi::LoadType::DEFAULT), - _approxByteSize(50) + _trace(), + _approxByteSize(50), + _priority(NORMAL) { } StorageMessage::StorageMessage(const StorageMessage& other, Id id) : _type(other._type), _msgId(id), - _priority(other._priority), _address(), - _loadType(other._loadType), - _approxByteSize(other._approxByteSize) + _trace(other.getTrace().getLevel()), + _approxByteSize(other._approxByteSize), + _priority(other._priority) { } StorageMessage::~StorageMessage() = default; -void StorageMessage::setNewMsgId() +const documentapi::LoadType& +StorageMessage::getLoadType() const { + return documentapi::LoadType::DEFAULT; +} + +void +StorageMessage::setNewMsgId() { _msgId = generateMsgId(); } @@ -298,7 +301,8 @@ StorageMessage::getSummary() const { return toString(); } -const char* to_string(LockingRequirements req) noexcept { +const char* +to_string(LockingRequirements req) noexcept { switch (req) { case LockingRequirements::Exclusive: return "Exclusive"; case LockingRequirements::Shared: return "Shared"; @@ -306,12 +310,14 @@ const char* to_string(LockingRequirements req) noexcept { } } -std::ostream& operator<<(std::ostream& os, LockingRequirements req) { +std::ostream& +operator<<(std::ostream& os, LockingRequirements req) { os << to_string(req); return os; } -const char* to_string(InternalReadConsistency consistency) noexcept { +const char* +to_string(InternalReadConsistency consistency) noexcept { switch (consistency) { case InternalReadConsistency::Strong: return "Strong"; case InternalReadConsistency::Weak: return "Weak"; @@ -319,7 +325,8 @@ const char* to_string(InternalReadConsistency consistency) noexcept { } } -std::ostream& operator<<(std::ostream& os, InternalReadConsistency consistency) { +std::ostream& +operator<<(std::ostream& os, InternalReadConsistency consistency) { os << to_string(consistency); return os; } diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index aa23d3a0cae..98552e473c1 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -12,7 +12,6 @@ #pragma once #include "messagehandler.h" -#include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/messagebus/routing/route.h> #include <vespa/messagebus/trace.h> #include <vespa/vdslib/state/nodetype.h> @@ -21,9 +20,8 @@ #include <map> #include <iosfwd> -namespace vespalib { - class asciistream; -} +namespace vespalib { class asciistream; } +namespace documentapi { class LoadType; } // The following macros are provided as a way to write storage messages simply. // They implement the parts of the code that can easily be automaticly // generated. @@ -268,17 +266,15 @@ public: enum Protocol { STORAGE, DOCUMENT }; private: - mbus::Route _route; - Protocol _protocol; + vespalib::string _cluster; // Used for internal VDS addresses only size_t _precomputed_storage_hash; - vespalib::string _cluster; const lib::NodeType* _type; + Protocol _protocol; uint16_t _index; public: StorageMessageAddress(); // Only to be used when transient default ctor semantics are needed by containers - StorageMessageAddress(const mbus::Route& route); StorageMessageAddress(vespalib::stringref clusterName, const lib::NodeType& type, uint16_t index, Protocol protocol = STORAGE); @@ -286,13 +282,13 @@ public: void setProtocol(Protocol p) { _protocol = p; } - const mbus::Route& getRoute() const { return _route; } + mbus::Route to_mbus_route() const; Protocol getProtocol() const { return _protocol; } uint16_t getIndex() const; const lib::NodeType& getNodeType() const; const vespalib::string& getCluster() const; - // Returns precomputed hash over <cluster, type, index> tuple. Other fields not included. + // Returns precomputed hash over <type, index> pair. Other fields not included. [[nodiscard]] size_t internal_storage_hash() const noexcept { return _precomputed_storage_hash; } @@ -359,12 +355,11 @@ protected: static Id generateMsgId(); const MessageType& _type; - Id _msgId; - Priority _priority; + Id _msgId; std::unique_ptr<StorageMessageAddress> _address; - documentapi::LoadType _loadType; - mbus::Trace _trace; - uint32_t _approxByteSize; + vespalib::Trace _trace; + uint32_t _approxByteSize; + Priority _priority; StorageMessage(const MessageType& code, Id id); StorageMessage(const StorageMessage&, Id id); @@ -373,7 +368,7 @@ protected: public: StorageMessage& operator=(const StorageMessage&) = delete; StorageMessage(const StorageMessage&) = delete; - virtual ~StorageMessage(); + ~StorageMessage() override; Id getMsgId() const { return _msgId; } @@ -394,7 +389,7 @@ public: const StorageMessageAddress* getAddress() const { return _address.get(); } void setAddress(const StorageMessageAddress& address) { - _address.reset(new StorageMessageAddress(address)); + _address = std::make_unique<StorageMessageAddress>(address); } /** @@ -432,16 +427,16 @@ public: */ virtual bool callHandler(MessageHandler&, const StorageMessage::SP&) const = 0; - const documentapi::LoadType& getLoadType() const { return _loadType; } - void setLoadType(const documentapi::LoadType& type) { _loadType = type; } + const documentapi::LoadType& getLoadType() const; + mbus::Trace && steal_trace() { return std::move(_trace); } mbus::Trace& getTrace() { return _trace; } const mbus::Trace& getTrace() const { return _trace; } /** Sets the trace object for this message. */ - void setTrace(const mbus::Trace &trace) { _trace = trace; } + void setTrace(vespalib::Trace && trace) { _trace = std::move(trace); } /** * Cheap version of tostring(). diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp index 81cdadb3623..2bb9fabd7d5 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp @@ -14,7 +14,12 @@ StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) if (cmd.getAddress()) { setAddress(*cmd.getAddress()); } - setTrace(cmd.getTrace()); + // TODD do we really need copy construction + if ( ! cmd.getTrace().isEmpty()) { + setTrace(vespalib::Trace(cmd.getTrace())); + } else { + getTrace().setLevel(cmd.getTrace().getLevel()); + } setTransportContext(cmd.getTransportContext()); } diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storageapi/src/vespa/storageapi/messageapi/storagereply.h index 1a3bbe35eb4..53516949110 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.h @@ -30,7 +30,7 @@ public: ~StorageReply() override; DECLARE_POINTER_TYPEDEFS(StorageReply); - void setResult(const ReturnCode& r) { _result = r; } + void setResult(ReturnCode r) { _result = std::move(r); } void setResult(ReturnCode::Result r) { _result = ReturnCode(r); } const ReturnCode& getResult() const { return _result; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index 2f165c69470..363dc6240dd 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -5,6 +5,7 @@ #include <vespa/storage/storageserver/servicelayernode.h> #include <vespa/storageserver/app/distributorprocess.h> #include <vespa/storageserver/app/dummyservicelayerprocess.h> +#include <vespa/messagebus/message.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> @@ -21,7 +22,7 @@ struct StorageServerTest : public ::testing::Test { std::unique_ptr<vdstestlib::DirConfig> storConfig; StorageServerTest(); - ~StorageServerTest(); + ~StorageServerTest() override; void SetUp() override; void TearDown() override; @@ -34,7 +35,7 @@ StorageServerTest::~StorageServerTest() = default; namespace { struct Node { - virtual ~Node() {} + virtual ~Node() = default; virtual StorageNode& getNode() = 0; virtual StorageNodeContext& getContext() = 0; }; @@ -43,10 +44,10 @@ struct Distributor : public Node { DistributorProcess _process; Distributor(vdstestlib::DirConfig& config); - ~Distributor(); + ~Distributor() override; - virtual StorageNode& getNode() override { return _process.getNode(); } - virtual StorageNodeContext& getContext() override { return _process.getContext(); } + StorageNode& getNode() override { return _process.getNode(); } + StorageNodeContext& getContext() override { return _process.getContext(); } }; struct Storage : public Node { @@ -54,10 +55,10 @@ struct Storage : public Node { StorageComponent::UP _component; Storage(vdstestlib::DirConfig& config); - ~Storage(); + ~Storage() override; - virtual StorageNode& getNode() override { return _process.getNode(); } - virtual StorageNodeContext& getContext() override { return _process.getContext(); } + StorageNode& getNode() override { return _process.getNode(); } + StorageNodeContext& getContext() override { return _process.getContext(); } }; Distributor::Distributor(vdstestlib::DirConfig& config) @@ -74,8 +75,8 @@ Storage::Storage(vdstestlib::DirConfig& config) { _process.setupConfig(60000ms); _process.createNode(); - _component.reset(new StorageComponent( - getContext().getComponentRegister(), "test")); + _component = std::make_unique<StorageComponent>( + getContext().getComponentRegister(), "test"); } Storage::~Storage() = default; @@ -87,9 +88,9 @@ StorageServerTest::SetUp() { [[maybe_unused]] int systemResult = system("chmod -R 755 vdsroot"); systemResult = system("rm -rf vdsroot*"); - slobrok.reset(new mbus::Slobrok); - distConfig.reset(new vdstestlib::DirConfig(getStandardConfig(false))); - storConfig.reset(new vdstestlib::DirConfig(getStandardConfig(true))); + slobrok = std::make_unique<mbus::Slobrok>(); + distConfig = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(false)); + storConfig = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true)); addSlobrokConfig(*distConfig, *slobrok); addSlobrokConfig(*storConfig, *slobrok); storConfig->getConfig("stor-filestor").set("fail_disk_after_error_count", "1"); diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index 8cfbfd204ba..7119bde7a09 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -4,6 +4,7 @@ package ai.vespa.hosted.plugin; import ai.vespa.hosted.api.ControllerHttpClient; import ai.vespa.hosted.api.Properties; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.yolean.Exceptions; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -15,6 +16,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; /** * Base class for hosted Vespa plugin mojos. @@ -61,7 +66,11 @@ public abstract class AbstractVespaMojo extends AbstractMojo { throw e; } catch (Exception e) { - throw new MojoExecutionException("Execution failed for application " + name(), e); + String message = "Execution failed for application " + name() + ":\n" + Exceptions.toMessageString(e); + if (e.getSuppressed().length > 0) + message += "\nSuppressed:\n" + Stream.of(e.getSuppressed()).map(Exceptions::toMessageString).collect(joining("\n")); + + throw new MojoExecutionException(message, e); } } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java b/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java index 37b560d1129..f5d5d1529bb 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/StripedExecutor.java @@ -68,7 +68,7 @@ public class StripedExecutor<Key> { command.run(); } catch (RuntimeException e) { - logger.log(Level.WARNING, () -> "Exception caught: " + Exceptions.toMessageString(e)); + logger.log(Level.WARNING, e, () -> "Exception caught: " + Exceptions.toMessageString(e)); } } } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java index d5235caef9f..5ab1c88775a 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java @@ -62,14 +62,13 @@ import java.util.List; * example. * </p> * - * @param AGGREGATOR - * the type input data is aggregated into - * @param SAMPLE - * the type of input data + * @param <AGGREGATOR> the type input data is aggregated into + * @param <SAMPLE> the type of input data * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { + /** * Factory interface to create the data container for each generation of * samples, and putting data into it. @@ -85,12 +84,11 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * need to implement both. * </p> * - * @param AGGREGATOR - * The type of the data container to produce - * @param SAMPLE - * The type of the incoming data to store in the container. + * @param <AGGREGATOR> the type of the data container to produce + * @param <SAMPLE> the type of the incoming data to store in the container. */ public interface Updater<AGGREGATOR, SAMPLE> { + /** * Create data container to receive produced data. This is invoked once * on every instance every time ThreadLocalDirectory.fetch() is invoked. @@ -137,7 +135,7 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * * @return a fresh structure to receive data */ - public AGGREGATOR createGenerationInstance(AGGREGATOR previous); + AGGREGATOR createGenerationInstance(AGGREGATOR previous); /** * Insert a data element of type S into the current generation of data @@ -180,7 +178,8 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * the data to insert * @return the new current value, may be the same as previous */ - public AGGREGATOR update(AGGREGATOR current, SAMPLE x); + AGGREGATOR update(AGGREGATOR current, SAMPLE x); + } /** @@ -188,14 +187,12 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * ThreadLocalDirectory without resetting the local instances in each * thread. * - * @param <AGGREGATOR> - * as for {@link Updater} - * @param <SAMPLE> - * as for {@link Updater} + * @param <AGGREGATOR> as for {@link Updater} + * @param <SAMPLE> as for {@link Updater} * @see ThreadLocalDirectory#view() */ - public interface ObservableUpdater<AGGREGATOR, SAMPLE> extends - Updater<AGGREGATOR, SAMPLE> { + public interface ObservableUpdater<AGGREGATOR, SAMPLE> extends Updater<AGGREGATOR, SAMPLE> { + /** * Create an application specific copy of the AGGREGATOR for a thread. * @@ -203,7 +200,8 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * the AGGREGATOR instance to copy * @return a copy of the incoming parameter */ - public AGGREGATOR copy(AGGREGATOR current); + AGGREGATOR copy(AGGREGATOR current); + } private final ThreadLocal<LocalInstance<AGGREGATOR, SAMPLE>> local = new ThreadLocal<>(); @@ -268,8 +266,7 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * to have been instantiated with an updater implementing ObservableUpdater. * * @return a list of a copy of the current data in all producer threads - * @throws IllegalStateException - * if the updater does not implement {@link ObservableUpdater} + * @throws IllegalStateException if the updater does not implement {@link ObservableUpdater} */ public List<AGGREGATOR> view() { if (observableUpdater == null) { @@ -310,8 +307,7 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { /** * Input data from a producer thread. * - * @param x - * the data to insert + * @param x the data to insert */ public void update(SAMPLE x) { update(x, getOrCreateLocal()); @@ -330,10 +326,8 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * calls necessary to update(SAMPLE, LocalInstance<AGGREGATOR, SAMPLE>). * </p> * - * @param x - * the data to insert - * @param localInstance - * the local data insertion instance + * @param x the data to insert + * @param localInstance the local data insertion instance */ public void update(SAMPLE x, LocalInstance<AGGREGATOR, SAMPLE> localInstance) { boolean isRegistered; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 242bdb1161e..2c123779a1e 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -47,6 +47,7 @@ public abstract class Maintainer implements Runnable, AutoCloseable { @Override public void run() { + log.log(Level.FINE, () -> "Running " + this.getClass().getSimpleName()); try { if (jobControl.isActive(name())) { lockAndMaintain(); @@ -56,6 +57,7 @@ public abstract class Maintainer implements Runnable, AutoCloseable { } catch (Throwable e) { log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e); } + log.log(Level.FINE, () -> "Finished " + this.getClass().getSimpleName()); } @Override diff --git a/vespalib/src/tests/spin_lock/spin_lock_test.cpp b/vespalib/src/tests/spin_lock/spin_lock_test.cpp index 5ba0ca16222..3542bd5d51f 100644 --- a/vespalib/src/tests/spin_lock/spin_lock_test.cpp +++ b/vespalib/src/tests/spin_lock/spin_lock_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <array> using namespace vespalib; diff --git a/vespalib/src/tests/trace/trace.cpp b/vespalib/src/tests/trace/trace.cpp index 92bee3231b0..992317b0289 100644 --- a/vespalib/src/tests/trace/trace.cpp +++ b/vespalib/src/tests/trace/trace.cpp @@ -146,25 +146,25 @@ TEST("testTraceLevel") t.setLevel(4); EXPECT_EQUAL(4u, t.getLevel()); t.trace(9, "no"); - EXPECT_EQUAL(0u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(0u, t.getNumChildren()); t.trace(8, "no"); - EXPECT_EQUAL(0u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(0u, t.getNumChildren()); t.trace(7, "no"); - EXPECT_EQUAL(0u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(0u, t.getNumChildren()); t.trace(6, "no"); - EXPECT_EQUAL(0u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(0u, t.getNumChildren()); t.trace(5, "no"); - EXPECT_EQUAL(0u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(0u, t.getNumChildren()); t.trace(4, "yes"); - EXPECT_EQUAL(1u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(1u, t.getNumChildren()); t.trace(3, "yes"); - EXPECT_EQUAL(2u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(2u, t.getNumChildren()); t.trace(2, "yes"); - EXPECT_EQUAL(3u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(3u, t.getNumChildren()); t.trace(1, "yes"); - EXPECT_EQUAL(4u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(4u, t.getNumChildren()); t.trace(0, "yes"); - EXPECT_EQUAL(5u, t.getRoot().getNumChildren()); + EXPECT_EQUAL(5u, t.getNumChildren()); } TEST("testCompact") @@ -261,10 +261,10 @@ TEST("testTraceDump") b1.addChild(b2); } for (int i = 0; i < 10; ++i) { - big.getRoot().addChild(b1); + big.addChild(TraceNode(b1)); } string normal = big.toString(); - string full = big.getRoot().toString(); + string full = big.toString(100000); EXPECT_GREATER(normal.size(), 30000u); EXPECT_LESS(normal.size(), 32000u); EXPECT_GREATER(full.size(), 50000u); diff --git a/vespalib/src/vespa/vespalib/data/slime/cursor.h b/vespalib/src/vespa/vespalib/data/slime/cursor.h index 6815ad3ba83..34d7028a027 100644 --- a/vespalib/src/vespa/vespalib/data/slime/cursor.h +++ b/vespalib/src/vespa/vespalib/data/slime/cursor.h @@ -5,8 +5,7 @@ #include "inspector.h" #include "external_memory.h" -namespace vespalib { -namespace slime { +namespace vespalib::slime { struct Cursor : public Inspector { virtual Cursor &operator[](size_t idx) const override = 0; @@ -46,6 +45,4 @@ struct Cursor : public Inspector { virtual Symbol resolve(Memory symbol_name) = 0; }; -} // namespace vespalib::slime -} // namespace vespalib - +} diff --git a/vespalib/src/vespa/vespalib/trace/trace.cpp b/vespalib/src/vespa/vespalib/trace/trace.cpp index 8ca2ef6561c..be370aebbd2 100644 --- a/vespalib/src/vespa/vespalib/trace/trace.cpp +++ b/vespalib/src/vespa/vespalib/trace/trace.cpp @@ -6,43 +6,13 @@ namespace vespalib { -Trace::Trace() : - _level(0), - _root() +Trace::Trace(const Trace &rhs) + : _root(), + _level(rhs._level) { - // empty -} - -Trace::Trace(uint32_t level) : - _level(level), - _root() -{ - // empty -} - -Trace::~Trace() = default; - -Trace & -Trace::clear() -{ - _level = 0; - _root.clear(); - return *this; -} - -Trace & -Trace::swap(Trace &other) -{ - std::swap(_level, other._level); - _root.swap(other._root); - return *this; -} - -Trace & -Trace::setLevel(uint32_t level) -{ - _level = std::min(level, 9u); - return *this; + if (!rhs.isEmpty()) { + _root = std::make_unique<TraceNode>(rhs.getRoot()); + } } bool @@ -53,12 +23,36 @@ Trace::trace(uint32_t level, const string ¬e, bool addTime) } if (addTime) { struct timeval tv; - gettimeofday(&tv, NULL); - _root.addChild(make_string("[%ld.%06ld] %s", tv.tv_sec, static_cast<long>(tv.tv_usec), note.c_str())); + gettimeofday(&tv, nullptr); + ensureRoot().addChild(make_string("[%ld.%06ld] %s", tv.tv_sec, static_cast<long>(tv.tv_usec), note.c_str())); } else { - _root.addChild(note); + ensureRoot().addChild(note); } return true; } +string +Trace::toString(size_t limit) const { + return _root ? _root->toString(limit) : ""; +} + +string +Trace::encode() const { + return isEmpty() ? "" : getRoot().encode(); +} + +void +Trace::clear() { + _level = 0; + _root.reset(); +} + +TraceNode & +Trace::ensureRoot() { + if (!_root) { + _root = std::make_unique<TraceNode>(); + } + return *_root; +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/trace/trace.h b/vespalib/src/vespa/vespalib/trace/trace.h index d1ca5c5d7e7..2f70785d77d 100644 --- a/vespalib/src/vespa/vespalib/trace/trace.h +++ b/vespalib/src/vespa/vespalib/trace/trace.h @@ -18,31 +18,23 @@ namespace vespalib { * information will be traced. */ class Trace { -private: - uint32_t _level; - TraceNode _root; - public: /** * Create an empty Trace with level set to 0 (no tracing) */ - Trace(); - ~Trace(); - - /** - * Create an empty trace with given level. - * - * @param level Level to set. - */ - explicit Trace(uint32_t level); + Trace() : Trace(0) {} + explicit Trace(uint32_t level) : _root(), _level(level) { } + Trace & operator = (Trace &&) = default; + Trace(Trace &&) = default; + Trace(const Trace &); + Trace & operator = (const Trace &) = delete; + ~Trace() = default; /** * Remove all trace information and set the trace level to 0. - * - * @return This, to allow chaining. */ - Trace &clear(); + void clear(); /** * Swap the internals of this with another. @@ -50,21 +42,16 @@ public: * @param other The trace to swap internals with. * @return This, to allow chaining. */ - Trace &swap(Trace &other); + Trace &swap(Trace &other) { + std::swap(_level, other._level); + _root.swap(other._root); + return *this; + } - /** - * Set the trace level. 0 means no tracing, 9 means enable all tracing. - * - * @param level The level to set. - * @return This, to allow chaining. - */ - Trace &setLevel(uint32_t level); + void setLevel(uint32_t level) { + _level = std::min(level, 9u); + } - /** - * Returns the trace level. - * - * @return The trace level. - */ uint32_t getLevel() const { return _level; } /** @@ -91,19 +78,29 @@ public: */ bool trace(uint32_t level, const string ¬e, bool addTime = true); - /** - * Returns the root of the trace tree. - * - * @return The root. - */ - TraceNode &getRoot() { return _root; } + void normalize() { + if (_root) { + _root->normalize(); + } + } - /** - * Returns a const reference to the root of the trace tree. - * - * @return The root. - */ - const TraceNode &getRoot() const { return _root; } + void setStrict(bool strict) { + ensureRoot().setStrict(strict); + } + void addChild(TraceNode && child) { + ensureRoot().addChild(std::move(child)); + } + void addChild(Trace && child) { + if (!child.isEmpty()) { + addChild(std::move(*child._root)); + } + } + + bool isEmpty() const { return !_root || _root->isEmpty(); } + + uint32_t getNumChildren() const { return _root ? _root->getNumChildren() : 0; } + const TraceNode & getChild(uint32_t child) const { return getRoot().getChild(child); } + string encode() const; /** * Returns a string representation of the contained trace tree. This is a @@ -111,7 +108,16 @@ public: * * @return Readable trace string. */ - string toString() const { return _root.toString(31337); } + string toString(size_t limit=31337) const; + size_t computeMemoryUsage() const { + return _root ? _root->computeMemoryUsage() : 0; + } +private: + const TraceNode &getRoot() const { return *_root; } + TraceNode &ensureRoot(); + + std::unique_ptr<TraceNode> _root; + uint32_t _level; }; #define VESPALIB_TRACE2(ttrace, level, note, addTime) \ diff --git a/vespalib/src/vespa/vespalib/trace/tracenode.cpp b/vespalib/src/vespa/vespalib/trace/tracenode.cpp index d34b45025d3..12dd51ac677 100644 --- a/vespalib/src/vespa/vespalib/trace/tracenode.cpp +++ b/vespalib/src/vespa/vespalib/trace/tracenode.cpp @@ -40,22 +40,22 @@ struct Cmp { } // namespace <unnamed> -TraceNode::TraceNode() : - _parent(nullptr), - _strict(true), - _hasNote(false), - _note(""), - _children(), - _timestamp() +TraceNode::TraceNode() + : _note(""), + _children(), + _parent(nullptr), + _timestamp(), + _strict(true), + _hasNote(false) { } -TraceNode::TraceNode(const TraceNode &rhs) : - _parent(nullptr), - _strict(rhs._strict), - _hasNote(rhs._hasNote), - _note(rhs._note), - _children(), - _timestamp(rhs._timestamp) +TraceNode::TraceNode(const TraceNode &rhs) + : _note(rhs._note), + _children(), + _parent(nullptr), + _timestamp(rhs._timestamp), + _strict(rhs._strict), + _hasNote(rhs._hasNote) { addChildren(rhs._children); } @@ -65,22 +65,22 @@ TraceNode & TraceNode::operator =(const TraceNode &) = default; TraceNode::~TraceNode() = default; -TraceNode::TraceNode(const string ¬e, system_time timestamp) : - _parent(nullptr), - _strict(true), - _hasNote(true), - _note(note), - _children(), - _timestamp(timestamp) +TraceNode::TraceNode(const string ¬e, system_time timestamp) + : _note(note), + _children(), + _parent(nullptr), + _timestamp(timestamp), + _strict(true), + _hasNote(true) { } -TraceNode::TraceNode(system_time timestamp) : - _parent(nullptr), - _strict(true), - _hasNote(false), - _note(""), - _children(), - _timestamp(timestamp) +TraceNode::TraceNode(system_time timestamp) + : _note(""), + _children(), + _parent(nullptr), + _timestamp(timestamp), + _strict(true), + _hasNote(false) { } TraceNode & @@ -342,4 +342,17 @@ TraceNode::accept(TraceVisitor & visitor) const return visitor; } +size_t +TraceNode::computeMemoryUsage() const +{ + if (isLeaf()) { + return getNote().size(); + } + size_t childSum = 0; + for (const TraceNode & child : _children) { + childSum += child.computeMemoryUsage(); + } + return childSum; +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/trace/tracenode.h b/vespalib/src/vespa/vespalib/trace/tracenode.h index 63e7bcd6dc0..7a7cdb89c69 100644 --- a/vespalib/src/vespa/vespalib/trace/tracenode.h +++ b/vespalib/src/vespa/vespalib/trace/tracenode.h @@ -23,12 +23,12 @@ struct TraceVisitor; */ class TraceNode { private: - TraceNode *_parent; - bool _strict; - bool _hasNote; string _note; std::vector<TraceNode> _children; + TraceNode *_parent; system_time _timestamp; + bool _strict; + bool _hasNote; public: /** @@ -253,6 +253,8 @@ public: */ TraceVisitor & accept(TraceVisitor & visitor) const; + size_t computeMemoryUsage() const; + }; } // namespace vespalib diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index e026559b283..e0de2622149 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -60,6 +60,7 @@ "com.yahoo.vespa.curator.Curator": { "superClass": "java.lang.Object", "interfaces": [ + "com.yahoo.vespa.curator.api.VespaCurator", "java.lang.AutoCloseable" ], "attributes": [ @@ -68,6 +69,7 @@ "methods": [ "public static com.yahoo.vespa.curator.Curator create(java.lang.String)", "public static com.yahoo.vespa.curator.Curator create(java.lang.String, java.util.Optional)", + "public void <init>(com.yahoo.cloud.config.CuratorConfig, com.yahoo.vespa.zookeeper.VespaZooKeeperServer)", "public void <init>(com.yahoo.cloud.config.ConfigserverConfig, com.yahoo.vespa.zookeeper.VespaZooKeeperServer)", "protected void <init>(java.lang.String, java.lang.String, java.util.function.Function)", "public java.lang.String connectionSpec()", @@ -89,7 +91,8 @@ "public org.apache.curator.framework.CuratorFramework framework()", "public void close()", "public java.lang.String zooKeeperEnsembleConnectionSpec()", - "public int zooKeeperEnsembleCount()" + "public int zooKeeperEnsembleCount()", + "public bridge synthetic java.lang.AutoCloseable lock(com.yahoo.path.Path, java.time.Duration)" ], "fields": [ "protected final org.apache.curator.RetryPolicy retryPolicy" @@ -110,5 +113,18 @@ "public void close()" ], "fields": [] + }, + "com.yahoo.vespa.curator.api.VespaCurator": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract java.lang.AutoCloseable lock(com.yahoo.path.Path, java.time.Duration)" + ], + "fields": [] } }
\ No newline at end of file diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/ConnectionSpec.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/ConnectionSpec.java new file mode 100644 index 00000000000..4409291419a --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/ConnectionSpec.java @@ -0,0 +1,102 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator; + +import com.yahoo.net.HostName; + +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +/** + * A connection spec for Curator. + * + * @author mpolden + */ +class ConnectionSpec { + + private final String local; + private final String ensemble; + private final int ensembleSize; + + private ConnectionSpec(String local, String ensemble, int ensembleSize) { + this.local = requireNonEmpty(local, "local spec"); + this.ensemble = requireNonEmpty(ensemble, "ensemble spec"); + this.ensembleSize = ensembleSize; + } + + /** Returns the local spec. This may be a subset of the ensemble spec */ + public String local() { + return local; + } + + /** Returns the ensemble spec. This always contains all nodes in the ensemble */ + public String ensemble() { + return ensemble; + } + + /** Returns the number of servers in the ensemble */ + public int ensembleSize() { + return ensembleSize; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectionSpec that = (ConnectionSpec) o; + return ensembleSize == that.ensembleSize && + local.equals(that.local) && + ensemble.equals(that.ensemble); + } + + @Override + public int hashCode() { + return Objects.hash(local, ensemble, ensembleSize); + } + + public static ConnectionSpec create(String spec) { + return create(spec, spec); + } + + public static ConnectionSpec create(String localSpec, String ensembleSpec) { + return new ConnectionSpec(localSpec, ensembleSpec, ensembleSpec.split(",").length); + } + + public static <T> ConnectionSpec create(List<T> servers, + Function<T, String> hostnameGetter, + Function<T, Integer> portGetter, + boolean localhostAffinity) { + String localSpec = createSpec(servers, hostnameGetter, portGetter, localhostAffinity); + String ensembleSpec = localhostAffinity ? createSpec(servers, hostnameGetter, portGetter, false) : localSpec; + return new ConnectionSpec(localSpec, ensembleSpec, servers.size()); + } + + private static <T> String createSpec(List<T> servers, + Function<T, String> hostnameGetter, + Function<T, Integer> portGetter, + boolean localhostAffinity) { + String thisServer = HostName.getLocalhost(); + StringBuilder connectionSpec = new StringBuilder(); + for (var server : servers) { + if (localhostAffinity && !thisServer.equals(hostnameGetter.apply(server))) continue; + connectionSpec.append(hostnameGetter.apply(server)); + connectionSpec.append(':'); + connectionSpec.append(portGetter.apply(server)); + connectionSpec.append(','); + } + if (localhostAffinity && connectionSpec.length() == 0) { + throw new IllegalArgumentException("Unable to create connect string to localhost: " + + "There is no localhost server specified in config"); + } + if (connectionSpec.length() > 0) { + connectionSpec.setLength(connectionSpec.length() - 1); // Remove trailing comma + } + return connectionSpec.toString(); + } + + private static String requireNonEmpty(String s, String field) { + if (Objects.requireNonNull(s).isEmpty()) throw new IllegalArgumentException(field + " must be non-empty"); + return s; + } + +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 04bd64219d4..90eec5760fc 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -3,10 +3,11 @@ package com.yahoo.vespa.curator; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.io.IOUtils; -import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.api.VespaCurator; import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.zookeeper.VespaZooKeeperServer; @@ -31,6 +32,7 @@ import java.io.File; import java.time.Duration; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -47,7 +49,7 @@ import java.util.logging.Logger; * @author vegardh * @author bratseth */ -public class Curator implements AutoCloseable { +public class Curator implements VespaCurator, AutoCloseable { private static final Logger LOG = Logger.getLogger(Curator.class.getName()); private static final File ZK_CLIENT_CONFIG_FILE = new File(Defaults.getDefaults().underVespaHome("conf/zookeeper/zookeeper-client.cfg")); @@ -55,81 +57,67 @@ public class Curator implements AutoCloseable { private static final Duration ZK_CONNECTION_TIMEOUT = Duration.ofSeconds(30); private static final Duration BASE_SLEEP_TIME = Duration.ofSeconds(1); private static final int MAX_RETRIES = 10; + private static final RetryPolicy DEFAULT_RETRY_POLICY = new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES); - protected final RetryPolicy retryPolicy; + protected final RetryPolicy retryPolicy = DEFAULT_RETRY_POLICY; private final CuratorFramework curatorFramework; - private final String connectionSpec; // May be a subset of the servers in the ensemble - private final String zooKeeperEnsembleConnectionSpec; - private final int zooKeeperEnsembleCount; + private final ConnectionSpec connectionSpec; // All lock keys, to allow re-entrancy. This will grow forever, but this should be too slow to be a problem private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>(); /** Creates a curator instance from a comma-separated string of ZooKeeper host:port strings */ public static Curator create(String connectionSpec) { - return new Curator(connectionSpec, connectionSpec, Optional.of(ZK_CLIENT_CONFIG_FILE)); + return new Curator(ConnectionSpec.create(connectionSpec), Optional.of(ZK_CLIENT_CONFIG_FILE)); } // For testing only, use Optional.empty for clientConfigFile parameter to create default zookeeper client config public static Curator create(String connectionSpec, Optional<File> clientConfigFile) { - return new Curator(connectionSpec, connectionSpec, clientConfigFile); + return new Curator(ConnectionSpec.create(connectionSpec), clientConfigFile); } - // Depend on ZooKeeperServer to make sure it is started first - // TODO: Move zookeeperserver config out of configserverconfig (requires update of controller services.xml as well) @Inject - public Curator(ConfigserverConfig configserverConfig, VespaZooKeeperServer server) { - this(configserverConfig, Optional.of(ZK_CLIENT_CONFIG_FILE)); + public Curator(CuratorConfig curatorConfig, @SuppressWarnings("unused") VespaZooKeeperServer server) { + // Depends on ZooKeeperServer to make sure it is started first + this(ConnectionSpec.create(curatorConfig.server(), + CuratorConfig.Server::hostname, + CuratorConfig.Server::port, + curatorConfig.zookeeperLocalhostAffinity()), + Optional.of(ZK_CLIENT_CONFIG_FILE)); } - Curator(ConfigserverConfig configserverConfig, Optional<File> clientConfigFile) { - this(createConnectionSpec(configserverConfig), createEnsembleConnectionSpec(configserverConfig), clientConfigFile); + // TODO: This can be removed when this package is no longer public API. + public Curator(ConfigserverConfig configserverConfig, @SuppressWarnings("unused") VespaZooKeeperServer server) { + this(ConnectionSpec.create(configserverConfig.zookeeperserver(), + ConfigserverConfig.Zookeeperserver::hostname, + ConfigserverConfig.Zookeeperserver::port, + configserverConfig.zookeeperLocalhostAffinity()), + Optional.of(ZK_CLIENT_CONFIG_FILE)); } - private Curator(String connectionSpec, String zooKeeperEnsembleConnectionSpec, Optional<File> clientConfigFile) { - this(connectionSpec, - zooKeeperEnsembleConnectionSpec, - (retryPolicy) -> CuratorFrameworkFactory - .builder() - .retryPolicy(retryPolicy) - .sessionTimeoutMs((int) ZK_SESSION_TIMEOUT.toMillis()) - .connectionTimeoutMs((int) ZK_CONNECTION_TIMEOUT.toMillis()) - .connectString(connectionSpec) - .zookeeperFactory(new VespaZooKeeperFactory(createClientConfig(clientConfigFile))) - .dontUseContainerParents() // TODO: Remove when we know ZooKeeper 3.5 works fine, consider waiting until Vespa 8 - .build()); - } - - protected Curator(String connectionSpec, - String zooKeeperEnsembleConnectionSpec, - Function<RetryPolicy, CuratorFramework> curatorFactory) { - this(connectionSpec, zooKeeperEnsembleConnectionSpec, curatorFactory, - new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES)); + protected Curator(String connectionSpec, String zooKeeperEnsembleConnectionSpec, Function<RetryPolicy, CuratorFramework> curatorFactory) { + this(ConnectionSpec.create(connectionSpec, zooKeeperEnsembleConnectionSpec), curatorFactory.apply(DEFAULT_RETRY_POLICY)); } - private Curator(String connectionSpec, - String zooKeeperEnsembleConnectionSpec, - Function<RetryPolicy, CuratorFramework> curatorFactory, - RetryPolicy retryPolicy) { - this.connectionSpec = connectionSpec; - this.retryPolicy = retryPolicy; - this.curatorFramework = curatorFactory.apply(retryPolicy); - if (this.curatorFramework != null) { - validateConnectionSpec(connectionSpec); - validateConnectionSpec(zooKeeperEnsembleConnectionSpec); - addLoggingListener(); - curatorFramework.start(); - } - - this.zooKeeperEnsembleConnectionSpec = zooKeeperEnsembleConnectionSpec; - this.zooKeeperEnsembleCount = zooKeeperEnsembleConnectionSpec.split(",").length; + Curator(ConnectionSpec connectionSpec, Optional<File> clientConfigFile) { + this(connectionSpec, + CuratorFrameworkFactory + .builder() + .retryPolicy(DEFAULT_RETRY_POLICY) + .sessionTimeoutMs((int) ZK_SESSION_TIMEOUT.toMillis()) + .connectionTimeoutMs((int) ZK_CONNECTION_TIMEOUT.toMillis()) + .connectString(connectionSpec.local()) + .zookeeperFactory(new VespaZooKeeperFactory(createClientConfig(clientConfigFile))) + .dontUseContainerParents() // TODO: Remove when we know ZooKeeper 3.5 works fine, consider waiting until Vespa 8 + .build()); } - private static String createConnectionSpec(ConfigserverConfig configserverConfig) { - return configserverConfig.zookeeperLocalhostAffinity() - ? createConnectionSpecForLocalhost(configserverConfig) - : createEnsembleConnectionSpec(configserverConfig); + private Curator(ConnectionSpec connectionSpec, CuratorFramework curatorFramework) { + this.connectionSpec = Objects.requireNonNull(connectionSpec); + this.curatorFramework = Objects.requireNonNull(curatorFramework); + addLoggingListener(); + curatorFramework.start(); } private static ZKClientConfig createClientConfig(Optional<File> clientConfigFile) { @@ -148,39 +136,6 @@ public class Curator implements AutoCloseable { } } - private static String createEnsembleConnectionSpec(ConfigserverConfig config) { - StringBuilder connectionSpec = new StringBuilder(); - for (int i = 0; i < config.zookeeperserver().size(); i++) { - if (connectionSpec.length() > 0) { - connectionSpec.append(','); - } - ConfigserverConfig.Zookeeperserver server = config.zookeeperserver(i); - connectionSpec.append(server.hostname()); - connectionSpec.append(':'); - connectionSpec.append(server.port()); - } - return connectionSpec.toString(); - } - - static String createConnectionSpecForLocalhost(ConfigserverConfig config) { - String thisServer = HostName.getLocalhost(); - - for (int i = 0; i < config.zookeeperserver().size(); i++) { - ConfigserverConfig.Zookeeperserver server = config.zookeeperserver(i); - if (thisServer.equals(server.hostname())) { - return String.format("%s:%d", server.hostname(), server.port()); - } - } - - throw new IllegalArgumentException("Unable to create connect string to localhost: " + - "There is no localhost server specified in config: " + config); - } - - private static void validateConnectionSpec(String connectionSpec) { - if (connectionSpec == null || connectionSpec.isEmpty()) - throw new IllegalArgumentException(String.format("Connections spec '%s' is not valid", connectionSpec)); - } - /** * Returns the ZooKeeper "connect string" used by curator: a comma-separated list of * host:port of ZooKeeper endpoints to connect to. This may be a subset of @@ -189,7 +144,7 @@ public class Curator implements AutoCloseable { * * This may be empty but never null */ - public String connectionSpec() { return connectionSpec; } + public String connectionSpec() { return connectionSpec.local(); } /** For internal use; prefer creating a {@link CuratorCounter} */ public DistributedAtomicLong createAtomicCounter(String path) { @@ -243,13 +198,14 @@ public class Curator implements AutoCloseable { * A convenience method which sets some content at a path. * If the path and any of its parents does not exists they are created. */ + // TODO: Use create().orSetData() in Curator 4 and later public void set(Path path, byte[] data) { + if ( ! exists(path)) + create(path); + String absolutePath = path.getAbsolute(); try { - if ( ! exists(path)) - framework().create().creatingParentsIfNeeded().forPath(absolutePath, data); - else - framework().setData().forPath(absolutePath, data); + framework().setData().forPath(absolutePath, data); } catch (Exception e) { throw new RuntimeException("Could not set data at " + absolutePath, e); } @@ -432,7 +388,7 @@ public class Curator implements AutoCloseable { * TODO: Move method out of this class. */ public String zooKeeperEnsembleConnectionSpec() { - return zooKeeperEnsembleConnectionSpec; + return connectionSpec.ensemble(); } /** @@ -440,7 +396,7 @@ public class Curator implements AutoCloseable { * WARNING: This may be different from the number of servers this Curator may connect to. * TODO: Move method out of this class. */ - public int zooKeeperEnsembleCount() { return zooKeeperEnsembleCount; } + public int zooKeeperEnsembleCount() { return connectionSpec.ensembleSize(); } private static Optional<String> getEnvironmentVariable(String variableName) { return Optional.ofNullable(System.getenv().get(variableName)) diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java index fb8c303db66..0f8c524fa98 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java @@ -57,6 +57,8 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { List<String> respondents; do { respondents = curator.framework().getChildren().forPath(barrierPath); + log.log(Level.FINE, respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + + respondents + ", all participants: " + curator.zooKeeperEnsembleConnectionSpec()); // First, check if all config servers responded if (respondents.size() == curator.zooKeeperEnsembleCount()) { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/api/VespaCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/api/VespaCurator.java new file mode 100644 index 00000000000..0b6fa55719e --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/api/VespaCurator.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.api; + +import com.yahoo.path.Path; + +import java.time.Duration; + +/** + * A client for a ZooKeeper cluster running inside Vespa. Applications that want to use ZooKeeper can inject this in + * their code. + * + * @author mpolden + */ +public interface VespaCurator { + + /** Create and acquire a re-entrant lock in given path. This blocks until the lock is acquired or timeout elapses. */ + AutoCloseable lock(Path path, Duration timeout); + +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/api/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/api/package-info.java new file mode 100644 index 00000000000..679dd3750cb --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/api/package-info.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author mpolden + */ +@PublicApi +@ExportPackage +package com.yahoo.vespa.curator.api; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index 1f583ada7a1..26f1c336874 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -1,110 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.mock; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; -import com.yahoo.collections.Pair; -import com.yahoo.concurrent.Lock; -import com.yahoo.concurrent.Locks; import com.yahoo.path.Path; -import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.recipes.CuratorLockException; -import org.apache.curator.CuratorZookeeperClient; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.WatcherRemoveCuratorFramework; -import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable; -import org.apache.curator.framework.api.ACLCreateModeBackgroundPathAndBytesable; -import org.apache.curator.framework.api.ACLCreateModePathAndBytesable; -import org.apache.curator.framework.api.ACLCreateModeStatBackgroundPathAndBytesable; -import org.apache.curator.framework.api.ACLPathAndBytesable; -import org.apache.curator.framework.api.ACLableExistBuilderMain; -import org.apache.curator.framework.api.BackgroundCallback; -import org.apache.curator.framework.api.BackgroundPathAndBytesable; -import org.apache.curator.framework.api.BackgroundPathable; -import org.apache.curator.framework.api.BackgroundVersionable; -import org.apache.curator.framework.api.ChildrenDeletable; -import org.apache.curator.framework.api.CreateBackgroundModeStatACLable; -import org.apache.curator.framework.api.CreateBuilder; -import org.apache.curator.framework.api.CreateBuilder2; -import org.apache.curator.framework.api.CreateBuilderMain; -import org.apache.curator.framework.api.CreateProtectACLCreateModePathAndBytesable; -import org.apache.curator.framework.api.CuratorListener; -import org.apache.curator.framework.api.CuratorWatcher; -import org.apache.curator.framework.api.DeleteBuilder; -import org.apache.curator.framework.api.DeleteBuilderMain; -import org.apache.curator.framework.api.ErrorListenerPathAndBytesable; -import org.apache.curator.framework.api.ErrorListenerPathable; -import org.apache.curator.framework.api.ExistsBuilder; -import org.apache.curator.framework.api.GetACLBuilder; -import org.apache.curator.framework.api.GetChildrenBuilder; -import org.apache.curator.framework.api.GetConfigBuilder; -import org.apache.curator.framework.api.GetDataBuilder; -import org.apache.curator.framework.api.GetDataWatchBackgroundStatable; -import org.apache.curator.framework.api.PathAndBytesable; -import org.apache.curator.framework.api.Pathable; -import org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable; -import org.apache.curator.framework.api.ReconfigBuilder; -import org.apache.curator.framework.api.RemoveWatchesBuilder; -import org.apache.curator.framework.api.SetACLBuilder; -import org.apache.curator.framework.api.SetDataBackgroundVersionable; -import org.apache.curator.framework.api.SetDataBuilder; -import org.apache.curator.framework.api.SyncBuilder; -import org.apache.curator.framework.api.UnhandledErrorListener; -import org.apache.curator.framework.api.VersionPathAndBytesable; -import org.apache.curator.framework.api.WatchPathable; -import org.apache.curator.framework.api.Watchable; -import org.apache.curator.framework.api.transaction.CuratorMultiTransaction; -import org.apache.curator.framework.api.transaction.CuratorTransaction; -import org.apache.curator.framework.api.transaction.CuratorTransactionBridge; -import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; -import org.apache.curator.framework.api.transaction.CuratorTransactionResult; -import org.apache.curator.framework.api.transaction.TransactionCheckBuilder; -import org.apache.curator.framework.api.transaction.TransactionCreateBuilder; -import org.apache.curator.framework.api.transaction.TransactionCreateBuilder2; -import org.apache.curator.framework.api.transaction.TransactionDeleteBuilder; -import org.apache.curator.framework.api.transaction.TransactionOp; -import org.apache.curator.framework.api.transaction.TransactionSetDataBuilder; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.listen.Listenable; -import org.apache.curator.framework.recipes.atomic.AtomicStats; -import org.apache.curator.framework.recipes.atomic.AtomicValue; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.NodeCacheListener; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessLock; -import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; -import org.apache.curator.framework.schema.SchemaSet; -import org.apache.curator.framework.state.ConnectionStateErrorPolicy; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.utils.EnsurePath; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import static com.yahoo.vespa.curator.mock.MemoryFileSystem.Node; /** * <p>A <b>non thread safe</b> mock of the curator API. @@ -121,24 +25,7 @@ import static com.yahoo.vespa.curator.mock.MemoryFileSystem.Node; */ public class MockCurator extends Curator { - public boolean timeoutOnLock = false; - public boolean throwExceptionOnLock = false; - private boolean shouldTimeoutOnEnter = false; - private int monotonicallyIncreasingNumber = 0; - private final boolean stableOrdering; private String zooKeeperEnsembleConnectionSpec = ""; - private final Locks<String> locks = new Locks<>(Long.MAX_VALUE, TimeUnit.DAYS); - - /** The file system used by this mock to store zookeeper files and directories */ - private final MemoryFileSystem fileSystem = new MemoryFileSystem(); - - /** Atomic counters. A more accurate mock would store these as files in the file system */ - private final Map<String, MockAtomicCounter> atomicCounters = new ConcurrentHashMap<>(); - - /** Listeners to changes to a particular path */ - private final ListenerMap listeners = new ListenerMap(); - - private final CuratorFramework curatorFramework; /** Creates a mock curator with stable ordering */ @Inject @@ -153,26 +40,24 @@ public class MockCurator extends Curator { * This is not what ZooKeeper does. */ public MockCurator(boolean stableOrdering) { - super("", "", (retryPolicy) -> null); - this.stableOrdering = stableOrdering; - curatorFramework = new MockCuratorFramework(); - curatorFramework.start(); + super("host1:2181", "host1:2181", (retryPolicy) -> new MockCuratorFramework(stableOrdering, false)); + } + + private MockCuratorFramework mockFramework() { + return (MockCuratorFramework) super.framework(); } /** * Lists the entire content of this curator instance as a multiline string. * Useful for debugging. */ - public String dumpState() { return fileSystem.dumpState(); } - - /** Returns a started curator framework */ - public CuratorFramework framework() { return curatorFramework; } + public String dumpState() { return mockFramework().fileSystem().dumpState(); } /** Returns an atomic counter in this, or empty if no such counter is created */ public Optional<DistributedAtomicLong> counter(String path) { - return Optional.ofNullable(atomicCounters.get(path)); + return Optional.ofNullable(mockFramework().atomicCounters().get(path)); } - + /** * Sets the ZooKeeper ensemble connection spec, which must be on the form * host1:port,host2:port ... @@ -186,1455 +71,37 @@ public class MockCurator extends Curator { return zooKeeperEnsembleConnectionSpec; } - // ----- Start of adaptor methods from Curator to the mock file system ----- - - /** Creates a node below the given directory root */ - private String createNode(String pathString, byte[] content, boolean createParents, CreateMode createMode, Node root, Listeners listeners) - throws KeeperException.NodeExistsException, KeeperException.NoNodeException { - validatePath(pathString); - Path path = Path.fromString(pathString); - if (path.isRoot()) return "/"; // the root already exists - Node parent = root.getNode(Paths.get(path.getParentPath().toString()), createParents); - String name = nodeName(path.getName(), createMode); - - if (parent == null) - throw new KeeperException.NoNodeException(path.getParentPath().toString()); - if (parent.children().containsKey(path.getName())) - throw new KeeperException.NodeExistsException(path.toString()); - - parent.add(name).setContent(content); - String nodePath = "/" + path.getParentPath().toString() + "/" + name; - listeners.notify(Path.fromString(nodePath), content, PathChildrenCacheEvent.Type.CHILD_ADDED); - return nodePath; - } - - /** Deletes a node below the given directory root */ - private void deleteNode(String pathString, boolean deleteChildren, Node root, Listeners listeners) - throws KeeperException.NoNodeException, KeeperException.NotEmptyException { - validatePath(pathString); - Path path = Path.fromString(pathString); - Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); - if (parent == null) throw new KeeperException.NoNodeException(path.toString()); - Node node = parent.children().get(path.getName()); - if (node == null) throw new KeeperException.NoNodeException(path.getName() + " under " + parent); - if ( ! node.children().isEmpty() && ! deleteChildren) - throw new KeeperException.NotEmptyException(path.toString()); - parent.remove(path.getName()); - listeners.notify(path, new byte[0], PathChildrenCacheEvent.Type.CHILD_REMOVED); - } - - /** Returns the data of a node */ - private byte[] getData(String pathString, Node root) throws KeeperException.NoNodeException { - validatePath(pathString); - return getNode(pathString, root).getContent(); - } - - /** sets the data of an existing node */ - private void setData(String pathString, byte[] content, Node root, Listeners listeners) - throws KeeperException.NoNodeException { - validatePath(pathString); - getNode(pathString, root).setContent(content); - listeners.notify(Path.fromString(pathString), content, PathChildrenCacheEvent.Type.CHILD_UPDATED); - } - - private List<String> getChildren(String path, Node root) throws KeeperException.NoNodeException { - validatePath(path); - Node node = root.getNode(Paths.get(path), false); - if (node == null) throw new KeeperException.NoNodeException(path); - List<String> children = new ArrayList<>(node.children().keySet()); - if (! stableOrdering) - Collections.shuffle(children); - return children; - } - - private boolean exists(String path, Node root) { - validatePath(path); - Node parent = root.getNode(Paths.get(Path.fromString(path).getParentPath().toString()), false); - if (parent == null) return false; - Node node = parent.children().get(Path.fromString(path).getName()); - return node != null; - } - - /** Returns a node or throws the appropriate exception if it doesn't exist */ - private Node getNode(String pathString, Node root) throws KeeperException.NoNodeException { - validatePath(pathString); - Path path = Path.fromString(pathString); - Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); - if (parent == null) throw new KeeperException.NoNodeException(path.toString()); - Node node = parent.children().get(path.getName()); - if (node == null) throw new KeeperException.NoNodeException(path.toString()); - return node; - } - - private String nodeName(String baseName, CreateMode createMode) { - switch (createMode) { - case PERSISTENT: case EPHEMERAL: return baseName; - case PERSISTENT_SEQUENTIAL: case EPHEMERAL_SEQUENTIAL: return baseName + monotonicallyIncreasingNumber++; - default: throw new UnsupportedOperationException(createMode + " support not implemented in MockCurator"); - } - } - - /** Validates a path using the same rules as ZooKeeper */ - public static String validatePath(String path) throws IllegalArgumentException { - if (path == null) throw new IllegalArgumentException("Path cannot be null"); - if (path.length() == 0) throw new IllegalArgumentException("Path length must be > 0"); - if (path.charAt(0) != '/') throw new IllegalArgumentException("Path must start with / character"); - if (path.length() == 1) return path; // done checking - it's the root - if (path.charAt(path.length() - 1) == '/') - throw new IllegalArgumentException("Path must not end with / character"); - - String reason = null; - char lastc = '/'; - char chars[] = path.toCharArray(); - char c; - for (int i = 1; i < chars.length; lastc = chars[i], i++) { - c = chars[i]; - - if (c == 0) { - reason = "null character not allowed @" + i; - break; - } else if (c == '/' && lastc == '/') { - reason = "empty node name specified @" + i; - break; - } else if (c == '.' && lastc == '.') { - if (chars[i-2] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { - reason = "relative paths not allowed @" + i; - break; - } - } else if (c == '.') { - if (chars[i-1] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { - reason = "relative paths not allowed @" + i; - break; - } - } else if (c > '\u0000' && c < '\u001f' || c > '\u007f' && c < '\u009F' - || c > '\ud800' && c < '\uf8ff' || c > '\ufff0' && c < '\uffff') { - reason = "invalid charater @" + i; - break; - } - } - - if (reason != null) - throw new IllegalArgumentException("Invalid path string \"" + path + "\" caused by " + reason); - return path; - } - - // ----- Mock of Curator recipes accessed through our Curator interface ----- - @Override public DistributedAtomicLong createAtomicCounter(String path) { - MockAtomicCounter counter = atomicCounters.get(path); - if (counter == null) { - counter = new MockAtomicCounter(path); - atomicCounters.put(path, counter); - } - return counter; + return mockFramework().createAtomicCounter(path); } - /** Create a mutex which ensures exclusive access within this single vm */ @Override public InterProcessLock createMutex(String path) { - return new MockLock(path); - } - - public MockCurator timeoutBarrierOnEnter(boolean shouldTimeout) { - shouldTimeoutOnEnter = shouldTimeout; - return this; + return mockFramework().createMutex(path); } @Override public CompletionWaiter getCompletionWaiter(Path parentPath, int numMembers, String id) { - return new MockCompletionWaiter(); + return mockFramework().createCompletionWaiter(); } @Override public CompletionWaiter createCompletionWaiter(Path parentPath, String waiterNode, int numMembers, String id) { - return new MockCompletionWaiter(); + return mockFramework().createCompletionWaiter(); } @Override public DirectoryCache createDirectoryCache(String path, boolean cacheData, boolean dataIsCompressed, ExecutorService executorService) { - return new MockDirectoryCache(Path.fromString(path)); + return mockFramework().createDirectoryCache(path); } @Override public FileCache createFileCache(String path, boolean dataIsCompressed) { - return new MockFileCache(Path.fromString(path)); + return mockFramework().createFileCache(path); } @Override public int zooKeeperEnsembleCount() { return 1; } - /** - * Invocation of changes to the file system state is abstracted through this to allow transactional - * changes to notify on commit - */ - private abstract class Listeners { - - /** Translating method */ - public final void notify(Path path, byte[] data, PathChildrenCacheEvent.Type type) { - String pathString = "/" + path.toString(); // this silly path class strips the leading "/" :-/ - PathChildrenCacheEvent event = new PathChildrenCacheEvent(type, new ChildData(pathString, null, data)); - notify(path, event); - } - - public abstract void notify(Path path, PathChildrenCacheEvent event); - - } - - /** The regular listener implementation which notifies registered file and directory listeners */ - private class ListenerMap extends Listeners { - - private final Map<Path, PathChildrenCacheListener> directoryListeners = new ConcurrentHashMap<>(); - private final Map<Path, NodeCacheListener> fileListeners = new ConcurrentHashMap<>(); - - public void add(Path path, PathChildrenCacheListener listener) { - directoryListeners.put(path, listener); - } - - public void add(Path path, NodeCacheListener listener) { - fileListeners.put(path, listener); - } - - @Override - public void notify(Path path, PathChildrenCacheEvent event) { - try { - // Snapshot directoryListeners in case notification leads to new directoryListeners added - Set<Map.Entry<Path, PathChildrenCacheListener>> directoryListenerSnapshot = new HashSet<>(directoryListeners.entrySet()); - for (Map.Entry<Path, PathChildrenCacheListener> listener : directoryListenerSnapshot) { - if (path.isChildOf(listener.getKey())) - listener.getValue().childEvent(curatorFramework, event); - } - - // Snapshot directoryListeners in case notification leads to new directoryListeners added - Set<Map.Entry<Path, NodeCacheListener>> fileListenerSnapshot = new HashSet<>(fileListeners.entrySet()); - for (Map.Entry<Path, NodeCacheListener> listener : fileListenerSnapshot) { - if (path.equals(listener.getKey())) - listener.getValue().nodeChanged(); - } - } - catch (Exception e) { - e.printStackTrace(); // TODO: Remove - throw new RuntimeException("Exception notifying listeners", e); - } - } - - } - - private class MockCompletionWaiter implements CompletionWaiter { - - @Override - public void awaitCompletion(Duration timeout) { - if (shouldTimeoutOnEnter) { - throw new CompletionTimeoutException(""); - } - } - - @Override - public void notifyCompletion() { - } - - } - - /** A lock which works inside a single vm */ - private class MockLock extends InterProcessSemaphoreMutex { - - private final String path; - - private Lock lock = null; - - public MockLock(String path) { - super(curatorFramework, path); - this.path = path; - } - - @Override - public boolean acquire(long timeout, TimeUnit unit) { - if (throwExceptionOnLock) - throw new CuratorLockException("Thrown by mock"); - if (timeoutOnLock) return false; - - try { - lock = locks.lock(path, timeout, unit); - return true; - } - catch (UncheckedTimeoutException e) { - return false; - } - } - - @Override - public void acquire() { - if (throwExceptionOnLock) - throw new CuratorLockException("Thrown by mock"); - - lock = locks.lock(path); - } - - @Override - public void release() { - if (lock != null) - lock.close(); - } - - } - - private class MockAtomicCounter extends DistributedAtomicLong { - - private boolean initialized = false; - private MockLongValue value = new MockLongValue(0); // yes, uninitialized returns 0 :-/ - - public MockAtomicCounter(String path) { - super(curatorFramework, path, retryPolicy); - } - - @Override - public boolean initialize(Long value) { - if (initialized) return false; - this.value = new MockLongValue(value); - initialized = true; - return true; - } - - @Override - public AtomicValue<Long> get() { - if (value == null) return new MockLongValue(0); - return value; - } - - public AtomicValue<Long> add(Long delta) throws Exception { - return trySet(value.postValue() + delta); - } - - public AtomicValue<Long> subtract(Long delta) throws Exception { - return trySet(value.postValue() - delta); - } - - @Override - public AtomicValue<Long> increment() { - return trySet(value.postValue() + 1); - } - - public AtomicValue<Long> decrement() throws Exception { - return trySet(value.postValue() - 1); - } - - @Override - public AtomicValue<Long> trySet(Long longval) { - value = new MockLongValue(longval); - return value; - } - - public void forceSet(Long newValue) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public AtomicValue<Long> compareAndSet(Long expectedValue, Long newValue) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - } - - private class MockLongValue implements AtomicValue<Long> { - - private AtomicLong value = new AtomicLong(); - - public MockLongValue(long value) { - this.value.set(value); - } - - @Override - public boolean succeeded() { - return true; - } - - public void setValue(long value) { - this.value.set(value); - } - - @Override - public Long preValue() { - return value.get(); - } - - @Override - public Long postValue() { - return value.get(); - } - - @Override - public AtomicStats getStats() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - } - - private class MockDirectoryCache implements DirectoryCache { - - /** The path this is caching and listening to */ - private Path path; - - public MockDirectoryCache(Path path) { - this.path = path; - } - - @Override - public void start() {} - - @Override - public void addListener(PathChildrenCacheListener listener) { - listeners.add(path, listener); - } - - @Override - public List<ChildData> getCurrentData() { - List<ChildData> childData = new ArrayList<>(); - for (String childName : getChildren(path)) { - Path childPath = path.append(childName); - childData.add(new ChildData(childPath.getAbsolute(), null, getData(childPath).get())); - } - return childData; - } - - @Override - public ChildData getCurrentData(Path fullPath) { - if (!fullPath.getParentPath().equals(path)) { - throw new IllegalArgumentException("Path '" + fullPath + "' is not a child path of '" + path + "'"); - } - - return getData(fullPath).map(bytes -> new ChildData(fullPath.getAbsolute(), null, bytes)).orElse(null); - } - - private void collectData(Node parent, Path parentPath, List<ChildData> data) { - for (Node child : parent.children().values()) { - Path childPath = parentPath.append(child.name()); - data.add(new ChildData("/" + childPath.toString(), null, child.getContent())); - } - } - - @Override - public void close() {} - - } - - private class MockFileCache implements FileCache { - - /** The path this is caching and listening to */ - private Path path; - - public MockFileCache(Path path) { - this.path = path; - } - - @Override - public void start() {} - - @Override - public void addListener(NodeCacheListener listener) { - listeners.add(path, listener); - } - - @Override - public ChildData getCurrentData() { - Node node = fileSystem.root().getNode(Paths.get(path.toString()), false); - if (node == null) return null; - return new ChildData("/" + path.toString(), null, node.getContent()); - } - - @Override - public void close() {} - - } - - // ----- The rest of this file is adapting the Curator (non-recipe) API to the ----- - // ----- file system methods above. ----- - // ----- There's nothing to see unless you are interested in an illustration of ----- - // ----- the folly of fluent API's or, more generally, mankind. ----- - private abstract static class MockProtectACLCreateModeStatPathAndBytesable<String> - implements ProtectACLCreateModeStatPathAndBytesable<String> { - - public BackgroundPathAndBytesable<String> withACL(List<ACL> list) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public BackgroundPathAndBytesable<String> withACL(List<ACL> list, boolean b) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public ProtectACLCreateModeStatPathAndBytesable<String> withMode(CreateMode createMode) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLCreateModeBackgroundPathAndBytesable<java.lang.String> withProtection() { - return null; - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground() { - return null; - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(Object o) { - return null; - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback) { - return null; - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o) { - return null; - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - return null; - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - return null; - } - - @Override - public ACLBackgroundPathAndBytesable<String> storingStatIn(Stat stat) { - return null; - } - - } - - private class MockCreateBuilder implements CreateBuilder { - - private boolean createParents = false; - private CreateMode createMode = CreateMode.PERSISTENT; - - @Override - public ProtectACLCreateModeStatPathAndBytesable<String> creatingParentsIfNeeded() { - createParents = true; - return new MockProtectACLCreateModeStatPathAndBytesable<>() { - - @Override - public String forPath(String s, byte[] bytes) throws Exception { - return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); - } - - @Override - public String forPath(String s) throws Exception { - return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); - } - - }; - } - - @Override - public ProtectACLCreateModeStatPathAndBytesable<String> creatingParentContainersIfNeeded() { - return new MockProtectACLCreateModeStatPathAndBytesable<>() { - - @Override - public String forPath(String s, byte[] bytes) throws Exception { - return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); - } - - @Override - public String forPath(String s) throws Exception { - return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); - } - - }; - } - - @Override - @Deprecated - public ACLPathAndBytesable<String> withProtectedEphemeralSequential() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLCreateModeStatBackgroundPathAndBytesable<String> withProtection() { - return null; - } - - public String forPath(String s) throws Exception { - return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); - } - - public String forPath(String s, byte[] bytes) throws Exception { - return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CreateBuilderMain withTtl(long l) { - return null; - } - - @Override - public CreateBuilder2 orSetData() { - return null; - } - - @Override - public CreateBuilder2 orSetData(int i) { - return null; - } - - @Override - public CreateBackgroundModeStatACLable compressed() { - return null; - } - - @Override - public CreateProtectACLCreateModePathAndBytesable<String> storingStatIn(Stat stat) { - return null; - } - - @Override - public BackgroundPathAndBytesable<String> withACL(List<ACL> list) { - return null; - } - - @Override - public ACLBackgroundPathAndBytesable<String> withMode(CreateMode createMode) { - this.createMode = createMode; - return this; - } - - @Override - public BackgroundPathAndBytesable<String> withACL(List<ACL> list, boolean b) { - return null; - } - } - - private static class MockBackgroundPathableBuilder<T> implements BackgroundPathable<T>, Watchable<BackgroundPathable<T>> { - - @Override - public ErrorListenerPathable<T> inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public T forPath(String s) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathable<T> watched() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathable<T> usingWatcher(Watcher watcher) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathable<T> usingWatcher(CuratorWatcher curatorWatcher) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - private class MockGetChildrenBuilder extends MockBackgroundPathableBuilder<List<String>> implements GetChildrenBuilder { - - @Override - public WatchPathable<List<String>> storingStatIn(Stat stat) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public List<String> forPath(String path) throws Exception { - return getChildren(path, fileSystem.root()); - } - - } - - private class MockExistsBuilder extends MockBackgroundPathableBuilder<Stat> implements ExistsBuilder { - - @Override - public Stat forPath(String path) throws Exception { - try { - Node node = getNode(path, fileSystem.root()); - Stat stat = new Stat(); - stat.setVersion(node.version()); - return stat; - } - catch (KeeperException.NoNodeException e) { - return null; - } - } - - @Override - public ACLableExistBuilderMain creatingParentsIfNeeded() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLableExistBuilderMain creatingParentContainersIfNeeded() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - private class MockDeleteBuilder extends MockBackgroundPathableBuilder<Void> implements DeleteBuilder { - - private boolean deleteChildren = false; - - @Override - public BackgroundVersionable deletingChildrenIfNeeded() { - deleteChildren = true; - return this; - } - - @Override - public ChildrenDeletable guaranteed() { - return this; - } - - @Override - public BackgroundPathable<Void> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public Void forPath(String pathString) throws Exception { - deleteNode(pathString, deleteChildren, fileSystem.root(), listeners); - return null; - } - - @Override - public DeleteBuilderMain quietly() { - return this; - } - } - - private class MockGetDataBuilder extends MockBackgroundPathableBuilder<byte[]> implements GetDataBuilder { - - @Override - public GetDataWatchBackgroundStatable decompressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public byte[] forPath(String path) throws Exception { - return getData(path, fileSystem.root()); - } - - @Override - public ErrorListenerPathable<byte[]> inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<byte[]> inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public WatchPathable<byte[]> storingStatIn(Stat stat) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - // extends MockBackgroundACLPathAndBytesableBuilder<Stat> - private class MockSetDataBuilder implements SetDataBuilder { - - @Override - public SetDataBackgroundVersionable compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathAndBytesable<Stat> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public Stat forPath(String path, byte[] bytes) throws Exception { - setData(path, bytes, fileSystem.root(), listeners); - return null; - } - - @Override - public Stat forPath(String s) throws Exception { - return null; - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - /** Allows addition of directoryListeners which are never called */ - private class MockListenable<T> implements Listenable<T> { - - @Override - public void addListener(T t) { - } - - @Override - public void addListener(T t, Executor executor) { - } - - @Override - public void removeListener(T t) { - } - - } - - private class MockCuratorTransactionFinal implements CuratorTransactionFinal { - - /** The new directory root in which the transactional changes are made */ - private Node newRoot; - - private boolean committed = false; - - private final DelayedListener delayedListener = new DelayedListener(); - - public MockCuratorTransactionFinal() { - newRoot = fileSystem.root().clone(); - } - - @Override - public Collection<CuratorTransactionResult> commit() throws Exception { - fileSystem.replaceRoot(newRoot); - committed = true; - delayedListener.commit(); - return null; // TODO - } - - @Override - public TransactionCreateBuilder create() { - ensureNotCommitted(); - return new MockTransactionCreateBuilder(); - } - - @Override - public TransactionDeleteBuilder delete() { - ensureNotCommitted(); - return new MockTransactionDeleteBuilder(); - } - - @Override - public TransactionSetDataBuilder setData() { - ensureNotCommitted(); - return new MockTransactionSetDataBuilder(); - } - - @Override - public TransactionCheckBuilder check() { - ensureNotCommitted(); - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - private void ensureNotCommitted() { - if (committed) throw new IllegalStateException("transaction already committed"); - } - - private class MockTransactionCreateBuilder implements TransactionCreateBuilder { - - private CreateMode createMode = CreateMode.PERSISTENT; - - @Override - public ACLCreateModePathAndBytesable<CuratorTransactionBridge> compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLPathAndBytesable<CuratorTransactionBridge> withMode(CreateMode createMode) { - this.createMode = createMode; - return this; - } - - @Override - public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { - createNode(s, bytes, false, createMode, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - @Override - public CuratorTransactionBridge forPath(String s) throws Exception { - createNode(s, new byte[0], false, createMode, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - @Override - public TransactionCreateBuilder2 withTtl(long l) { - return this; - } - - @Override - public Object withACL(List list, boolean b) { - return this; - } - - @Override - public Object withACL(List list) { - return this; - } - } - - private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { - - @Override - public Pathable<CuratorTransactionBridge> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorTransactionBridge forPath(String path) throws Exception { - deleteNode(path, false, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - } - - private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder { - - @Override - public VersionPathAndBytesable<CuratorTransactionBridge> compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public PathAndBytesable<CuratorTransactionBridge> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { - MockCurator.this.setData(s, bytes, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - @Override - public CuratorTransactionBridge forPath(String s) throws Exception { - MockCurator.this.setData(s, new byte[0], newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - } - - private class MockCuratorTransactionBridge implements CuratorTransactionBridge { - - @Override - public CuratorTransactionFinal and() { - return MockCuratorTransactionFinal.this; - } - - } - - /** A class which collects listen events and forwards them to the regular directoryListeners on commit */ - private class DelayedListener extends Listeners { - - private final List<Pair<Path, PathChildrenCacheEvent>> events = new ArrayList<>(); - - @Override - public void notify(Path path, PathChildrenCacheEvent event) { - events.add(new Pair<>(path, event)); - } - - public void commit() { - for (Pair<Path, PathChildrenCacheEvent> event : events) - listeners.notify(event.getFirst(), event.getSecond()); - } - - } - - } - - private class MockCuratorFramework implements CuratorFramework { - - private CuratorFrameworkState curatorState = CuratorFrameworkState.LATENT; - - @Override - public void start() { - curatorState = CuratorFrameworkState.STARTED; - } - - @Override - public void close() { - curatorState = CuratorFrameworkState.STOPPED; - } - - @Override - public CuratorFrameworkState getState() { - return curatorState; - } - - @Override - @Deprecated - public boolean isStarted() { - return curatorState == CuratorFrameworkState.STARTED; - } - - @Override - public CreateBuilder create() { - return new MockCreateBuilder(); - } - - @Override - public DeleteBuilder delete() { - return new MockDeleteBuilder(); - } - - @Override - public ExistsBuilder checkExists() { - return new MockExistsBuilder(); - } - - @Override - public GetDataBuilder getData() { - return new MockGetDataBuilder(); - } - - @Override - public SetDataBuilder setData() { - return new MockSetDataBuilder(); - } - - @Override - public GetChildrenBuilder getChildren() { - return new MockGetChildrenBuilder(); - } - - @Override - public GetACLBuilder getACL() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public SetACLBuilder setACL() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ReconfigBuilder reconfig() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public GetConfigBuilder getConfig() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorTransaction inTransaction() { - return new MockCuratorTransactionFinal(); - } - - @Override - public CuratorMultiTransaction transaction() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public TransactionOp transactionOp() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - @Deprecated - public void sync(String path, Object backgroundContextObject) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public void createContainers(String s) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public Listenable<ConnectionStateListener> getConnectionStateListenable() { - return new MockListenable<>(); - } - - @Override - public Listenable<CuratorListener> getCuratorListenable() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public Listenable<UnhandledErrorListener> getUnhandledErrorListenable() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - @Deprecated - public CuratorFramework nonNamespaceView() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorFramework usingNamespace(String newNamespace) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public String getNamespace() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorZookeeperClient getZookeeperClient() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Deprecated - @Override - public EnsurePath newNamespaceAwareEnsurePath(String path) { - return new EnsurePath(path); - } - - @Override - public void clearWatcherReferences(Watcher watcher) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public boolean blockUntilConnected(int i, TimeUnit timeUnit) throws InterruptedException { - return true; - } - - @Override - public void blockUntilConnected() throws InterruptedException { - - } - - @Override - public WatcherRemoveCuratorFramework newWatcherRemoveCuratorFramework() { - return new WatcherRemoveCuratorFramework() { - @Override - public void removeWatchers() { - - } - - @Override - public void start() { - - } - - @Override - public void close() { - - } - - @Override - public CuratorFrameworkState getState() { - return null; - } - - @Override - public boolean isStarted() { - return false; - } - - @Override - public CreateBuilder create() { - return null; - } - - @Override - public DeleteBuilder delete() { - return null; - } - - @Override - public ExistsBuilder checkExists() { - return null; - } - - @Override - public GetDataBuilder getData() { - return null; - } - - @Override - public SetDataBuilder setData() { - return null; - } - - @Override - public GetChildrenBuilder getChildren() { - return null; - } - - @Override - public GetACLBuilder getACL() { - return null; - } - - @Override - public SetACLBuilder setACL() { - return null; - } - - @Override - public ReconfigBuilder reconfig() { - return null; - } - - @Override - public GetConfigBuilder getConfig() { - return null; - } - - @Override - public CuratorTransaction inTransaction() { - return null; - } - - @Override - public CuratorMultiTransaction transaction() { - return null; - } - - @Override - public TransactionOp transactionOp() { - return null; - } - - @Override - public void sync(String s, Object o) { - - } - - @Override - public void createContainers(String s) throws Exception { - - } - - @Override - public SyncBuilder sync() { - return null; - } - - @Override - public RemoveWatchesBuilder watches() { - return null; - } - - @Override - public Listenable<ConnectionStateListener> getConnectionStateListenable() { - return null; - } - - @Override - public Listenable<CuratorListener> getCuratorListenable() { - return null; - } - - @Override - public Listenable<UnhandledErrorListener> getUnhandledErrorListenable() { - return null; - } - - @Override - public CuratorFramework nonNamespaceView() { - return null; - } - - @Override - public CuratorFramework usingNamespace(String s) { - return null; - } - - @Override - public String getNamespace() { - return null; - } - - @Override - public CuratorZookeeperClient getZookeeperClient() { - return null; - } - - @Override - public EnsurePath newNamespaceAwareEnsurePath(String s) { - return null; - } - - @Override - public void clearWatcherReferences(Watcher watcher) { - - } - - @Override - public boolean blockUntilConnected(int i, TimeUnit timeUnit) throws InterruptedException { - return false; - } - - @Override - public void blockUntilConnected() throws InterruptedException { - - } - - @Override - public WatcherRemoveCuratorFramework newWatcherRemoveCuratorFramework() { - return null; - } - - @Override - public ConnectionStateErrorPolicy getConnectionStateErrorPolicy() { - return null; - } - - @Override - public QuorumVerifier getCurrentConfig() { - return null; - } - - @Override - public SchemaSet getSchemaSet() { - return null; - } - - @Override - public boolean isZk34CompatibilityMode() { - return false; - } - - @Override - public CompletableFuture<Void> runSafe(Runnable runnable) { - return null; - } - }; - - } - - @Override - public ConnectionStateErrorPolicy getConnectionStateErrorPolicy() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public QuorumVerifier getCurrentConfig() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public SchemaSet getSchemaSet() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public boolean isZk34CompatibilityMode() { - return false; - } - - @Override - public CompletableFuture<Void> runSafe(Runnable runnable) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public SyncBuilder sync() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public RemoveWatchesBuilder watches() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - } - } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java new file mode 100644 index 00000000000..401c6e90cd8 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -0,0 +1,1356 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.mock; + +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.collections.Pair; +import com.yahoo.concurrent.Lock; +import com.yahoo.concurrent.Locks; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.CompletionTimeoutException; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorLockException; +import org.apache.curator.CuratorZookeeperClient; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.WatcherRemoveCuratorFramework; +import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable; +import org.apache.curator.framework.api.ACLCreateModeBackgroundPathAndBytesable; +import org.apache.curator.framework.api.ACLCreateModePathAndBytesable; +import org.apache.curator.framework.api.ACLCreateModeStatBackgroundPathAndBytesable; +import org.apache.curator.framework.api.ACLPathAndBytesable; +import org.apache.curator.framework.api.ACLableExistBuilderMain; +import org.apache.curator.framework.api.BackgroundCallback; +import org.apache.curator.framework.api.BackgroundPathAndBytesable; +import org.apache.curator.framework.api.BackgroundPathable; +import org.apache.curator.framework.api.BackgroundVersionable; +import org.apache.curator.framework.api.ChildrenDeletable; +import org.apache.curator.framework.api.CreateBackgroundModeStatACLable; +import org.apache.curator.framework.api.CreateBuilder; +import org.apache.curator.framework.api.CreateBuilder2; +import org.apache.curator.framework.api.CreateBuilderMain; +import org.apache.curator.framework.api.CreateProtectACLCreateModePathAndBytesable; +import org.apache.curator.framework.api.CuratorListener; +import org.apache.curator.framework.api.CuratorWatcher; +import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.DeleteBuilderMain; +import org.apache.curator.framework.api.ErrorListenerPathAndBytesable; +import org.apache.curator.framework.api.ErrorListenerPathable; +import org.apache.curator.framework.api.ExistsBuilder; +import org.apache.curator.framework.api.GetACLBuilder; +import org.apache.curator.framework.api.GetChildrenBuilder; +import org.apache.curator.framework.api.GetConfigBuilder; +import org.apache.curator.framework.api.GetDataBuilder; +import org.apache.curator.framework.api.GetDataWatchBackgroundStatable; +import org.apache.curator.framework.api.PathAndBytesable; +import org.apache.curator.framework.api.Pathable; +import org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable; +import org.apache.curator.framework.api.ReconfigBuilder; +import org.apache.curator.framework.api.RemoveWatchesBuilder; +import org.apache.curator.framework.api.SetACLBuilder; +import org.apache.curator.framework.api.SetDataBackgroundVersionable; +import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.curator.framework.api.SyncBuilder; +import org.apache.curator.framework.api.UnhandledErrorListener; +import org.apache.curator.framework.api.VersionPathAndBytesable; +import org.apache.curator.framework.api.WatchPathable; +import org.apache.curator.framework.api.Watchable; +import org.apache.curator.framework.api.transaction.CuratorMultiTransaction; +import org.apache.curator.framework.api.transaction.CuratorTransaction; +import org.apache.curator.framework.api.transaction.CuratorTransactionBridge; +import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; +import org.apache.curator.framework.api.transaction.CuratorTransactionResult; +import org.apache.curator.framework.api.transaction.TransactionCheckBuilder; +import org.apache.curator.framework.api.transaction.TransactionCreateBuilder; +import org.apache.curator.framework.api.transaction.TransactionCreateBuilder2; +import org.apache.curator.framework.api.transaction.TransactionDeleteBuilder; +import org.apache.curator.framework.api.transaction.TransactionOp; +import org.apache.curator.framework.api.transaction.TransactionSetDataBuilder; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.listen.Listenable; +import org.apache.curator.framework.recipes.atomic.AtomicStats; +import org.apache.curator.framework.recipes.atomic.AtomicValue; +import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.NodeCacheListener; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; +import org.apache.curator.framework.schema.SchemaSet; +import org.apache.curator.framework.state.ConnectionStateErrorPolicy; +import org.apache.curator.framework.state.ConnectionStateListener; +import org.apache.curator.retry.RetryForever; +import org.apache.curator.utils.EnsurePath; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; + +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A mock implementation of{@link CuratorFramework} for testing purposes. + * + * @author mpolden + */ +public class MockCuratorFramework implements CuratorFramework { + + private final boolean shouldTimeoutOnEnter; + private final boolean stableOrdering; + private final Locks<String> locks = new Locks<>(Long.MAX_VALUE, TimeUnit.DAYS); + + /** The file system used by this mock to store zookeeper files and directories */ + private final MemoryFileSystem fileSystem = new MemoryFileSystem(); + + /** Atomic counters. A more accurate mock would store these as files in the file system */ + private final Map<String, MockAtomicCounter> atomicCounters = new ConcurrentHashMap<>(); + + /** Listeners to changes to a particular path */ + private final ListenerMap listeners = new ListenerMap(); + + private CuratorFrameworkState curatorState = CuratorFrameworkState.LATENT; + private int monotonicallyIncreasingNumber = 0; + + public MockCuratorFramework(boolean stableOrdering, boolean shouldTimeoutOnEnter) { + this.stableOrdering = stableOrdering; + this.shouldTimeoutOnEnter = shouldTimeoutOnEnter; + } + + public Map<String, MockAtomicCounter> atomicCounters() { + return Collections.unmodifiableMap(atomicCounters); + } + + public MemoryFileSystem fileSystem() { + return fileSystem; + } + + @Override + public void start() { + curatorState = CuratorFrameworkState.STARTED; + } + + @Override + public void close() { + curatorState = CuratorFrameworkState.STOPPED; + } + + @Override + public CuratorFrameworkState getState() { + return curatorState; + } + + @Override + @Deprecated + public boolean isStarted() { + return curatorState == CuratorFrameworkState.STARTED; + } + + @Override + public CreateBuilder create() { + return new MockCreateBuilder(); + } + + @Override + public DeleteBuilder delete() { + return new MockDeleteBuilder(); + } + + @Override + public ExistsBuilder checkExists() { + return new MockExistsBuilder(); + } + + @Override + public GetDataBuilder getData() { + return new MockGetDataBuilder(); + } + + @Override + public SetDataBuilder setData() { + return new MockSetDataBuilder(); + } + + @Override + public GetChildrenBuilder getChildren() { + return new MockGetChildrenBuilder(); + } + + @Override + public GetACLBuilder getACL() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public SetACLBuilder setACL() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + + @Override + public ReconfigBuilder reconfig() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public GetConfigBuilder getConfig() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public CuratorTransaction inTransaction() { + return new MockCuratorTransactionFinal(); + } + + @Override + public CuratorMultiTransaction transaction() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public TransactionOp transactionOp() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public RemoveWatchesBuilder watches() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public WatcherRemoveCuratorFramework newWatcherRemoveCuratorFramework() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public ConnectionStateErrorPolicy getConnectionStateErrorPolicy() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public QuorumVerifier getCurrentConfig() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public SchemaSet getSchemaSet() { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + public boolean isZk34CompatibilityMode() { return false; } + + @Override + public CompletableFuture<Void> runSafe(Runnable runnable) { throw new UnsupportedOperationException("Not implemented in MockCurator"); } + + @Override + @Deprecated + public void sync(String path, Object backgroundContextObject) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public void createContainers(String s) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public Listenable<ConnectionStateListener> getConnectionStateListenable() { + return new MockListenable<>(); + } + + @Override + public Listenable<CuratorListener> getCuratorListenable() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public Listenable<UnhandledErrorListener> getUnhandledErrorListenable() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + @Deprecated + public CuratorFramework nonNamespaceView() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorFramework usingNamespace(String newNamespace) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public String getNamespace() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorZookeeperClient getZookeeperClient() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Deprecated + @Override + public EnsurePath newNamespaceAwareEnsurePath(String path) { + return new EnsurePath(path); + } + + @Override + public void clearWatcherReferences(Watcher watcher) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public boolean blockUntilConnected(int i, TimeUnit timeUnit) { + return true; + } + + @Override + public void blockUntilConnected() { + } + + @Override + public SyncBuilder sync() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + // ----- Factory methods for mocks */ + + public InterProcessLock createMutex(String path) { + return new MockLock(path); + } + + public MockAtomicCounter createAtomicCounter(String path) { + return atomicCounters.computeIfAbsent(path, (k) -> new MockAtomicCounter(path)); + } + + public Curator.CompletionWaiter createCompletionWaiter() { + return new MockCompletionWaiter(); + } + + public Curator.DirectoryCache createDirectoryCache(String path) { + return new MockDirectoryCache(Path.fromString(path)); + } + + public Curator.FileCache createFileCache(String path) { + return new MockFileCache(Path.fromString(path)); + } + + // ----- Start of adaptor methods from Curator to the mock file system ----- + + /** Creates a node below the given directory root */ + private String createNode(String pathString, byte[] content, boolean createParents, CreateMode createMode, MemoryFileSystem.Node root, Listeners listeners) + throws KeeperException.NodeExistsException, KeeperException.NoNodeException { + validatePath(pathString); + Path path = Path.fromString(pathString); + if (path.isRoot()) return "/"; // the root already exists + MemoryFileSystem.Node parent = root.getNode(Paths.get(path.getParentPath().toString()), createParents); + String name = nodeName(path.getName(), createMode); + + if (parent == null) + throw new KeeperException.NoNodeException(path.getParentPath().toString()); + if (parent.children().containsKey(path.getName())) + throw new KeeperException.NodeExistsException(path.toString()); + + parent.add(name).setContent(content); + String nodePath = "/" + path.getParentPath().toString() + "/" + name; + listeners.notify(Path.fromString(nodePath), content, PathChildrenCacheEvent.Type.CHILD_ADDED); + return nodePath; + } + + /** Deletes a node below the given directory root */ + private void deleteNode(String pathString, boolean deleteChildren, MemoryFileSystem.Node root, Listeners listeners) + throws KeeperException.NoNodeException, KeeperException.NotEmptyException { + validatePath(pathString); + Path path = Path.fromString(pathString); + MemoryFileSystem.Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); + if (parent == null) throw new KeeperException.NoNodeException(path.toString()); + MemoryFileSystem.Node node = parent.children().get(path.getName()); + if (node == null) throw new KeeperException.NoNodeException(path.getName() + " under " + parent); + if ( ! node.children().isEmpty() && ! deleteChildren) + throw new KeeperException.NotEmptyException(path.toString()); + parent.remove(path.getName()); + listeners.notify(path, new byte[0], PathChildrenCacheEvent.Type.CHILD_REMOVED); + } + + /** Returns the data of a node */ + private byte[] getData(String pathString, MemoryFileSystem.Node root) throws KeeperException.NoNodeException { + validatePath(pathString); + return getNode(pathString, root).getContent(); + } + + /** sets the data of an existing node */ + private void setData(String pathString, byte[] content, MemoryFileSystem.Node root, Listeners listeners) + throws KeeperException.NoNodeException { + validatePath(pathString); + getNode(pathString, root).setContent(content); + listeners.notify(Path.fromString(pathString), content, PathChildrenCacheEvent.Type.CHILD_UPDATED); + } + + private List<String> getChildren(String path, MemoryFileSystem.Node root) throws KeeperException.NoNodeException { + validatePath(path); + MemoryFileSystem.Node node = root.getNode(Paths.get(path), false); + if (node == null) throw new KeeperException.NoNodeException(path); + List<String> children = new ArrayList<>(node.children().keySet()); + if (! stableOrdering) + Collections.shuffle(children); + return children; + } + + /** Returns a node or throws the appropriate exception if it doesn't exist */ + private MemoryFileSystem.Node getNode(String pathString, MemoryFileSystem.Node root) throws KeeperException.NoNodeException { + validatePath(pathString); + Path path = Path.fromString(pathString); + MemoryFileSystem.Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); + if (parent == null) throw new KeeperException.NoNodeException(path.toString()); + MemoryFileSystem.Node node = parent.children().get(path.getName()); + if (node == null) throw new KeeperException.NoNodeException(path.toString()); + return node; + } + + private String nodeName(String baseName, CreateMode createMode) { + switch (createMode) { + case PERSISTENT: case EPHEMERAL: return baseName; + case PERSISTENT_SEQUENTIAL: case EPHEMERAL_SEQUENTIAL: return baseName + monotonicallyIncreasingNumber++; + default: throw new UnsupportedOperationException(createMode + " support not implemented in MockCurator"); + } + } + + /** Validates a path using the same rules as ZooKeeper */ + public static String validatePath(String path) throws IllegalArgumentException { + if (path == null) throw new IllegalArgumentException("Path cannot be null"); + if (path.length() == 0) throw new IllegalArgumentException("Path length must be > 0"); + if (path.charAt(0) != '/') throw new IllegalArgumentException("Path must start with / character"); + if (path.length() == 1) return path; // done checking - it's the root + if (path.charAt(path.length() - 1) == '/') + throw new IllegalArgumentException("Path must not end with / character"); + + String reason = null; + char lastc = '/'; + char[] chars = path.toCharArray(); + char c; + for (int i = 1; i < chars.length; lastc = chars[i], i++) { + c = chars[i]; + + if (c == 0) { + reason = "null character not allowed @" + i; + break; + } else if (c == '/' && lastc == '/') { + reason = "empty node name specified @" + i; + break; + } else if (c == '.' && lastc == '.') { + if (chars[i-2] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { + reason = "relative paths not allowed @" + i; + break; + } + } else if (c == '.') { + if (chars[i-1] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { + reason = "relative paths not allowed @" + i; + break; + } + } else if (c > '\u0000' && c < '\u001f' || c > '\u007f' && c < '\u009F' + || c > '\ud800' && c < '\uf8ff' || c > '\ufff0' && c < '\uffff') { + reason = "invalid charater @" + i; + break; + } + } + + if (reason != null) + throw new IllegalArgumentException("Invalid path string \"" + path + "\" caused by " + reason); + return path; + } + + /** + * Invocation of changes to the file system state is abstracted through this to allow transactional + * changes to notify on commit + */ + private abstract static class Listeners { + + /** Translating method */ + public final void notify(Path path, byte[] data, PathChildrenCacheEvent.Type type) { + String pathString = "/" + path.toString(); // this silly path class strips the leading "/" :-/ + PathChildrenCacheEvent event = new PathChildrenCacheEvent(type, new ChildData(pathString, null, data)); + notify(path, event); + } + + public abstract void notify(Path path, PathChildrenCacheEvent event); + + } + + /** The regular listener implementation which notifies registered file and directory listeners */ + private class ListenerMap extends Listeners { + + private final Map<Path, PathChildrenCacheListener> directoryListeners = new ConcurrentHashMap<>(); + private final Map<Path, NodeCacheListener> fileListeners = new ConcurrentHashMap<>(); + + public void add(Path path, PathChildrenCacheListener listener) { + directoryListeners.put(path, listener); + } + + public void add(Path path, NodeCacheListener listener) { + fileListeners.put(path, listener); + } + + @Override + public void notify(Path path, PathChildrenCacheEvent event) { + try { + // Snapshot directoryListeners in case notification leads to new directoryListeners added + Set<Map.Entry<Path, PathChildrenCacheListener>> directoryListenerSnapshot = new HashSet<>(directoryListeners.entrySet()); + for (Map.Entry<Path, PathChildrenCacheListener> listener : directoryListenerSnapshot) { + if (path.isChildOf(listener.getKey())) + listener.getValue().childEvent(MockCuratorFramework.this, event); + } + + // Snapshot directoryListeners in case notification leads to new directoryListeners added + Set<Map.Entry<Path, NodeCacheListener>> fileListenerSnapshot = new HashSet<>(fileListeners.entrySet()); + for (Map.Entry<Path, NodeCacheListener> listener : fileListenerSnapshot) { + if (path.equals(listener.getKey())) + listener.getValue().nodeChanged(); + } + } + catch (Exception e) { + e.printStackTrace(); // TODO: Remove + throw new RuntimeException("Exception notifying listeners", e); + } + } + + } + + private class MockCompletionWaiter implements Curator.CompletionWaiter { + + @Override + public void awaitCompletion(Duration timeout) { + if (shouldTimeoutOnEnter) { + throw new CompletionTimeoutException(""); + } + } + + @Override + public void notifyCompletion() { + } + + } + + /** A lock which works inside a single vm */ + private class MockLock extends InterProcessSemaphoreMutex { + + public boolean timeoutOnLock = false; + public boolean throwExceptionOnLock = false; + + private final String path; + + private Lock lock = null; + + public MockLock(String path) { + super(MockCuratorFramework.this, path); + this.path = path; + } + + @Override + public boolean acquire(long timeout, TimeUnit unit) { + if (throwExceptionOnLock) + throw new CuratorLockException("Thrown by mock"); + if (timeoutOnLock) return false; + + try { + lock = locks.lock(path, timeout, unit); + return true; + } + catch (UncheckedTimeoutException e) { + return false; + } + } + + @Override + public void acquire() { + if (throwExceptionOnLock) + throw new CuratorLockException("Thrown by mock"); + + lock = locks.lock(path); + } + + @Override + public void release() { + if (lock != null) + lock.close(); + } + + } + + private class MockAtomicCounter extends DistributedAtomicLong { + + private boolean initialized = false; + private MockLongValue value = new MockLongValue(0); // yes, uninitialized returns 0 :-/ + + public MockAtomicCounter(String path) { + super(MockCuratorFramework.this, path, new RetryForever(1_000)); + } + + @Override + public boolean initialize(Long value) { + if (initialized) return false; + this.value = new MockLongValue(value); + initialized = true; + return true; + } + + @Override + public AtomicValue<Long> get() { + if (value == null) return new MockLongValue(0); + return value; + } + + public AtomicValue<Long> add(Long delta) { + return trySet(value.postValue() + delta); + } + + public AtomicValue<Long> subtract(Long delta) { + return trySet(value.postValue() - delta); + } + + @Override + public AtomicValue<Long> increment() { + return trySet(value.postValue() + 1); + } + + public AtomicValue<Long> decrement() { + return trySet(value.postValue() - 1); + } + + @Override + public AtomicValue<Long> trySet(Long longval) { + value = new MockLongValue(longval); + return value; + } + + public void forceSet(Long newValue) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public AtomicValue<Long> compareAndSet(Long expectedValue, Long newValue) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + } + + private static class MockLongValue implements AtomicValue<Long> { + + private final AtomicLong value = new AtomicLong(); + + public MockLongValue(long value) { + this.value.set(value); + } + + @Override + public boolean succeeded() { + return true; + } + + public void setValue(long value) { + this.value.set(value); + } + + @Override + public Long preValue() { + return value.get(); + } + + @Override + public Long postValue() { + return value.get(); + } + + @Override + public AtomicStats getStats() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + } + + private class MockDirectoryCache implements Curator.DirectoryCache { + + /** The path this is caching and listening to */ + private final Path path; + + public MockDirectoryCache(Path path) { + this.path = path; + } + + @Override + public void start() {} + + @Override + public void addListener(PathChildrenCacheListener listener) { + listeners.add(path, listener); + } + + @Override + public List<ChildData> getCurrentData() { + List<ChildData> childData = new ArrayList<>(); + for (String childName : getChildren(path)) { + Path childPath = path.append(childName); + childData.add(new ChildData(childPath.getAbsolute(), null, getData(childPath).get())); + } + return childData; + } + + @Override + public ChildData getCurrentData(Path fullPath) { + if (!fullPath.getParentPath().equals(path)) { + throw new IllegalArgumentException("Path '" + fullPath + "' is not a child path of '" + path + "'"); + } + + return getData(fullPath).map(bytes -> new ChildData(fullPath.getAbsolute(), null, bytes)).orElse(null); + } + + @Override + public void close() {} + + private List<String> getChildren(Path path) { + try { + return MockCuratorFramework.this.getChildren().forPath(path.getAbsolute()); + } catch (KeeperException.NoNodeException e) { + return List.of(); + } catch (Exception e) { + throw new RuntimeException("Could not get children of " + path.getAbsolute(), e); + } + } + + private Optional<byte[]> getData(Path path) { + try { + return Optional.of(MockCuratorFramework.this.getData().forPath(path.getAbsolute())); + } + catch (KeeperException.NoNodeException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new RuntimeException("Could not get data at " + path.getAbsolute(), e); + } + } + + } + + private class MockFileCache implements Curator.FileCache { + + /** The path this is caching and listening to */ + private final Path path; + + public MockFileCache(Path path) { + this.path = path; + } + + @Override + public void start() {} + + @Override + public void addListener(NodeCacheListener listener) { + listeners.add(path, listener); + } + + @Override + public ChildData getCurrentData() { + MemoryFileSystem.Node node = fileSystem.root().getNode(Paths.get(path.toString()), false); + if (node == null) return null; + return new ChildData("/" + path.toString(), null, node.getContent()); + } + + @Override + public void close() {} + + } + + + // ----- The rest of this file is adapting the Curator (non-recipe) API to the ----- + // ----- file system methods above. ----- + // ----- There's nothing to see unless you are interested in an illustration of ----- + // ----- the folly of fluent API's or, more generally, mankind. ----- + private abstract static class MockProtectACLCreateModeStatPathAndBytesable<String> + implements ProtectACLCreateModeStatPathAndBytesable<String> { + + public BackgroundPathAndBytesable<String> withACL(List<ACL> list) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public BackgroundPathAndBytesable<String> withACL(List<ACL> list, boolean b) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public ProtectACLCreateModeStatPathAndBytesable<String> withMode(CreateMode createMode) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLCreateModeBackgroundPathAndBytesable<java.lang.String> withProtection() { + return null; + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground() { + return null; + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(Object o) { + return null; + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback) { + return null; + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o) { + return null; + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + return null; + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + return null; + } + + @Override + public ACLBackgroundPathAndBytesable<String> storingStatIn(Stat stat) { + return null; + } + + } + + private class MockCreateBuilder implements CreateBuilder { + + private boolean createParents = false; + private CreateMode createMode = CreateMode.PERSISTENT; + + @Override + public ProtectACLCreateModeStatPathAndBytesable<String> creatingParentsIfNeeded() { + createParents = true; + return new MockProtectACLCreateModeStatPathAndBytesable<>() { + + @Override + public String forPath(String s, byte[] bytes) throws Exception { + return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); + } + + @Override + public String forPath(String s) throws Exception { + return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); + } + + }; + } + + @Override + public ProtectACLCreateModeStatPathAndBytesable<String> creatingParentContainersIfNeeded() { + return new MockProtectACLCreateModeStatPathAndBytesable<>() { + + @Override + public String forPath(String s, byte[] bytes) throws Exception { + return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); + } + + @Override + public String forPath(String s) throws Exception { + return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); + } + + }; + } + + @Override + @Deprecated + public ACLPathAndBytesable<String> withProtectedEphemeralSequential() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLCreateModeStatBackgroundPathAndBytesable<String> withProtection() { + return null; + } + + public String forPath(String s) throws Exception { + return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); + } + + public String forPath(String s, byte[] bytes) throws Exception { + return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CreateBuilderMain withTtl(long l) { + return null; + } + + @Override + public CreateBuilder2 orSetData() { + return null; + } + + @Override + public CreateBuilder2 orSetData(int i) { + return null; + } + + @Override + public CreateBackgroundModeStatACLable compressed() { + return null; + } + + @Override + public CreateProtectACLCreateModePathAndBytesable<String> storingStatIn(Stat stat) { + return null; + } + + @Override + public BackgroundPathAndBytesable<String> withACL(List<ACL> list) { + return null; + } + + @Override + public ACLBackgroundPathAndBytesable<String> withMode(CreateMode createMode) { + this.createMode = createMode; + return this; + } + + @Override + public BackgroundPathAndBytesable<String> withACL(List<ACL> list, boolean b) { + return null; + } + } + + private static class MockBackgroundPathableBuilder<T> implements BackgroundPathable<T>, Watchable<BackgroundPathable<T>> { + + @Override + public ErrorListenerPathable<T> inBackground() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public T forPath(String s) throws Exception { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathable<T> watched() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathable<T> usingWatcher(Watcher watcher) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathable<T> usingWatcher(CuratorWatcher curatorWatcher) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + private class MockGetChildrenBuilder extends MockBackgroundPathableBuilder<List<String>> implements GetChildrenBuilder { + + @Override + public WatchPathable<List<String>> storingStatIn(Stat stat) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public List<String> forPath(String path) throws Exception { + return getChildren(path, fileSystem.root()); + } + + } + + private class MockExistsBuilder extends MockBackgroundPathableBuilder<Stat> implements ExistsBuilder { + + @Override + public Stat forPath(String path) throws Exception { + try { + MemoryFileSystem.Node node = getNode(path, fileSystem.root()); + Stat stat = new Stat(); + stat.setVersion(node.version()); + return stat; + } + catch (KeeperException.NoNodeException e) { + return null; + } + } + + @Override + public ACLableExistBuilderMain creatingParentsIfNeeded() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLableExistBuilderMain creatingParentContainersIfNeeded() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + private class MockDeleteBuilder extends MockBackgroundPathableBuilder<Void> implements DeleteBuilder { + + private boolean deleteChildren = false; + + @Override + public BackgroundVersionable deletingChildrenIfNeeded() { + deleteChildren = true; + return this; + } + + @Override + public ChildrenDeletable guaranteed() { + return this; + } + + @Override + public BackgroundPathable<Void> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public Void forPath(String pathString) throws Exception { + deleteNode(pathString, deleteChildren, fileSystem.root(), listeners); + return null; + } + + @Override + public DeleteBuilderMain quietly() { + return this; + } + } + + private class MockGetDataBuilder extends MockBackgroundPathableBuilder<byte[]> implements GetDataBuilder { + + @Override + public GetDataWatchBackgroundStatable decompressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public byte[] forPath(String path) throws Exception { + return getData(path, fileSystem.root()); + } + + @Override + public ErrorListenerPathable<byte[]> inBackground() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<byte[]> inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<byte[]> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public WatchPathable<byte[]> storingStatIn(Stat stat) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + // extends MockBackgroundACLPathAndBytesableBuilder<Stat> + private class MockSetDataBuilder implements SetDataBuilder { + + @Override + public SetDataBackgroundVersionable compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathAndBytesable<Stat> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public Stat forPath(String path, byte[] bytes) throws Exception { + setData(path, bytes, fileSystem.root(), listeners); + return null; + } + + @Override + public Stat forPath(String s) throws Exception { + return null; + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + /** Allows addition of directoryListeners which are never called */ + private class MockListenable<T> implements Listenable<T> { + + @Override + public void addListener(T t) { + } + + @Override + public void addListener(T t, Executor executor) { + } + + @Override + public void removeListener(T t) { + } + + } + + private class MockCuratorTransactionFinal implements CuratorTransactionFinal { + + /** The new directory root in which the transactional changes are made */ + private MemoryFileSystem.Node newRoot; + + private boolean committed = false; + + private final DelayedListener delayedListener = new DelayedListener(); + + public MockCuratorTransactionFinal() { + newRoot = fileSystem.root().clone(); + } + + @Override + public Collection<CuratorTransactionResult> commit() throws Exception { + fileSystem.replaceRoot(newRoot); + committed = true; + delayedListener.commit(); + return null; // TODO + } + + @Override + public TransactionCreateBuilder create() { + ensureNotCommitted(); + return new MockTransactionCreateBuilder(); + } + + @Override + public TransactionDeleteBuilder delete() { + ensureNotCommitted(); + return new MockTransactionDeleteBuilder(); + } + + @Override + public TransactionSetDataBuilder setData() { + ensureNotCommitted(); + return new MockTransactionSetDataBuilder(); + } + + @Override + public TransactionCheckBuilder check() { + ensureNotCommitted(); + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + private void ensureNotCommitted() { + if (committed) throw new IllegalStateException("transaction already committed"); + } + + private class MockTransactionCreateBuilder implements TransactionCreateBuilder { + + private CreateMode createMode = CreateMode.PERSISTENT; + + @Override + public ACLCreateModePathAndBytesable<CuratorTransactionBridge> compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLPathAndBytesable<CuratorTransactionBridge> withMode(CreateMode createMode) { + this.createMode = createMode; + return this; + } + + @Override + public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { + createNode(s, bytes, false, createMode, newRoot, delayedListener); + return new MockCuratorTransactionBridge(); + } + + @Override + public CuratorTransactionBridge forPath(String s) throws Exception { + createNode(s, new byte[0], false, createMode, newRoot, delayedListener); + return new MockCuratorTransactionBridge(); + } + + @Override + public TransactionCreateBuilder2 withTtl(long l) { + return this; + } + + @Override + public Object withACL(List list, boolean b) { + return this; + } + + @Override + public Object withACL(List list) { + return this; + } + } + + private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { + + @Override + public Pathable<CuratorTransactionBridge> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorTransactionBridge forPath(String path) throws Exception { + deleteNode(path, false, newRoot, delayedListener); + return new MockCuratorTransactionBridge(); + } + + } + + private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder { + + @Override + public VersionPathAndBytesable<CuratorTransactionBridge> compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public PathAndBytesable<CuratorTransactionBridge> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { + MockCuratorFramework.this.setData(s, bytes, newRoot, delayedListener); + return new MockCuratorTransactionBridge(); + } + + @Override + public CuratorTransactionBridge forPath(String s) throws Exception { + MockCuratorFramework.this.setData(s, new byte[0], newRoot, delayedListener); + return new MockCuratorTransactionBridge(); + } + + } + + private class MockCuratorTransactionBridge implements CuratorTransactionBridge { + + @Override + public CuratorTransactionFinal and() { + return MockCuratorTransactionFinal.this; + } + + } + + /** A class which collects listen events and forwards them to the regular directoryListeners on commit */ + private class DelayedListener extends Listeners { + + private final List<Pair<Path, PathChildrenCacheEvent>> events = new ArrayList<>(); + + @Override + public void notify(Path path, PathChildrenCacheEvent event) { + events.add(new Pair<>(path, event)); + } + + public void commit() { + for (Pair<Path, PathChildrenCacheEvent> event : events) + listeners.notify(event.getFirst(), event.getSecond()); + } + + } + + } + +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java index 07f0924cb31..777da5988eb 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/package-info.java @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi +@PublicApi // TODO: Revoke this on Vespa 8. package com.yahoo.vespa.curator; import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/ConnectionSpecTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/ConnectionSpecTest.java new file mode 100644 index 00000000000..a518d8df843 --- /dev/null +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/ConnectionSpecTest.java @@ -0,0 +1,74 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator; + +import com.yahoo.net.HostName; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ConnectionSpecTest { + + @Test + public void create() { + HostName.setHostNameForTestingOnly("host2"); + Config config = new Config(List.of(new Config.Server("host1", 10001), + new Config.Server("host2", 10002), + new Config.Server("host3", 10003))); + + { + ConnectionSpec spec = ConnectionSpec.create(config.servers, Config.Server::hostname, Config.Server::port, false); + assertEquals("host1:10001,host2:10002,host3:10003", spec.local()); + assertEquals("host1:10001,host2:10002,host3:10003", spec.ensemble()); + assertEquals(3, spec.ensembleSize()); + } + + { + ConnectionSpec specLocalAffinity = ConnectionSpec.create(config.servers, Config.Server::hostname, Config.Server::port, true); + assertEquals("host2:10002", specLocalAffinity.local()); + assertEquals("host1:10001,host2:10002,host3:10003", specLocalAffinity.ensemble()); + assertEquals(3, specLocalAffinity.ensembleSize()); + } + + { + ConnectionSpec specFromString = ConnectionSpec.create("host1:10001", "host1:10001,host2:10002"); + assertEquals("host1:10001", specFromString.local()); + assertEquals("host1:10001,host2:10002", specFromString.ensemble()); + assertEquals(2, specFromString.ensembleSize()); + } + } + + private static class Config { + + private final List<Server> servers; + + public Config(List<Server> servers) { + this.servers = servers; + } + + private static class Server { + + private final String hostname; + private final int port; + + public Server(String hostname, int port) { + this.hostname = hostname; + this.port = port; + } + + public String hostname() { + return hostname; + } + + public int port() { + return port; + } + } + + } + +} diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java index 2bf40c4e2bb..5341efaefe5 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; -import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.net.HostName; import org.apache.curator.test.TestingServer; import org.junit.After; @@ -61,45 +61,33 @@ public class CuratorTest { @Test public void require_that_server_count_is_correct() { - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - builder.zookeeperserver(createZKBuilder(localhost, port1)); - try (Curator curator = createCurator(new ConfigserverConfig(builder))) { + CuratorConfig.Builder builder = new CuratorConfig.Builder(); + builder.server(createZKBuilder(localhost, port1)); + try (Curator curator = createCurator(new CuratorConfig(builder))) { assertEquals(1, curator.zooKeeperEnsembleCount()); } } - @Test - public void localhost_affinity() { - String localhostHostName = "myhost"; - int localhostPort = 123; - - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - builder.zookeeperserver(createZKBuilder(localhostHostName, localhostPort)); - builder.zookeeperserver(createZKBuilder("otherhost", 345)); - ConfigserverConfig config = new ConfigserverConfig(builder); - - HostName.setHostNameForTestingOnly(localhostHostName); - - String localhostSpec = localhostHostName + ":" + localhostPort; - assertEquals(localhostSpec, Curator.createConnectionSpecForLocalhost(config)); - } - - private ConfigserverConfig createTestConfig() { - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - builder.zookeeperserver(createZKBuilder(localhost, port1)); - builder.zookeeperserver(createZKBuilder(localhost, port2)); - return new ConfigserverConfig(builder); + private CuratorConfig createTestConfig() { + CuratorConfig.Builder builder = new CuratorConfig.Builder(); + builder.server(createZKBuilder(localhost, port1)); + builder.server(createZKBuilder(localhost, port2)); + return new CuratorConfig(builder); } - private ConfigserverConfig.Zookeeperserver.Builder createZKBuilder(String hostname, int port) { - ConfigserverConfig.Zookeeperserver.Builder zkBuilder = new ConfigserverConfig.Zookeeperserver.Builder(); + private CuratorConfig.Server.Builder createZKBuilder(String hostname, int port) { + CuratorConfig.Server.Builder zkBuilder = new CuratorConfig.Server.Builder(); zkBuilder.hostname(hostname); zkBuilder.port(port); return zkBuilder; } - private Curator createCurator(ConfigserverConfig configserverConfig) { - return new Curator(configserverConfig, Optional.empty()); + private Curator createCurator(CuratorConfig curatorConfig) { + return new Curator(ConnectionSpec.create(curatorConfig.server(), + CuratorConfig.Server::hostname, + CuratorConfig.Server::port, + curatorConfig.zookeeperLocalhostAffinity()), + Optional.empty()); } private static class PortAllocator { diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index ee3695b02f8..3c42d881ecc 100644 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -1,55 +1,28 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.AbstractComponent; -import com.yahoo.security.tls.TransportSecurityUtils; - -import java.util.logging.Level; - -import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import static com.yahoo.vespa.zookeeper.Configurator.zookeeperServerHostnames; /** - * Writes zookeeper config and starts zookeeper server. + * Main component controlling startup and stop of zookeeper server * * @author Ulf Lilleengen * @author Harald Musum */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer { - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName()); - private final Thread zkServerThread; - private final ZookeeperServerConfig zookeeperServerConfig; +public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { + + private final ZooKeeperRunner zooKeeperRunner; @Inject public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this.zookeeperServerConfig = zookeeperServerConfig; - new Configurator(zookeeperServerConfig).writeConfigToDisk(TransportSecurityUtils.getOptions()); - zkServerThread = new Thread(this, "zookeeper server"); - zkServerThread.start(); - } - - private void shutdown() { - zkServerThread.interrupt(); - try { - zkServerThread.join(); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Error joining server thread on shutdown", e); - } - } - - @Override - public void run() { - String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; - log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + - ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); - org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); + this.zooKeeperRunner = new ZooKeeperRunner(zookeeperServerConfig); } @Override public void deconstruct() { - shutdown(); + zooKeeperRunner.shutdown(); super.deconstruct(); } diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java new file mode 100644 index 00000000000..492417cef96 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperRunner.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.security.tls.TransportSecurityUtils; + +import java.util.logging.Level; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static com.yahoo.vespa.zookeeper.Configurator.zookeeperServerHostnames; + +/** + * Writes zookeeper config and starts zookeeper server. + * + * @author Harald Musum + */ +public class ZooKeeperRunner implements Runnable { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ZooKeeperRunner.class.getName()); + private final Thread zkServerThread; + private final ZookeeperServerConfig zookeeperServerConfig; + + public ZooKeeperRunner(ZookeeperServerConfig zookeeperServerConfig) { + this.zookeeperServerConfig = zookeeperServerConfig; + new Configurator(zookeeperServerConfig).writeConfigToDisk(TransportSecurityUtils.getOptions()); + zkServerThread = new Thread(this, "zookeeper server"); + zkServerThread.start(); + } + + void shutdown() { + zkServerThread.interrupt(); + try { + zkServerThread.join(); + } catch (InterruptedException e) { + log.log(Level.WARNING, "Error joining server thread on shutdown", e); + } + } + + @Override + public void run() { + String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; + log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + + ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); + org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); + } + + public ZookeeperServerConfig zookeeperServerConfig() { + return zookeeperServerConfig; + } +} |