diff options
420 files changed, 9221 insertions, 3445 deletions
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index 8e5622e6c2f..f92f02f7908 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -63,7 +63,7 @@ public class IdentityDocumentGeneratorTest { Optional.empty(), new MockNodeFlavors().getFlavorOrThrow("default"), Optional.empty(), - NodeType.host); + NodeType.host, Optional.empty()); Node containerNode = Node.createDockerNode(Set.of("::1"), containerHostname, parentHostname, diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index ab81cb8eda5..5e965761874 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -231,7 +231,7 @@ public class InstanceValidatorTest { Optional.empty(), flavors.getFlavorOrThrow("default"), Optional.empty(), - NodeType.tenant); + NodeType.tenant, Optional.empty()); nodeList.add(node); } return nodeList; diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java index 5f6cc252d85..91f08fb9943 100644 --- a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/cloud/impl/VespaTestRuntime.java @@ -54,7 +54,14 @@ public class VespaTestRuntime implements TestRuntime { private static TestConfig configFromPropertyOrController() { String configPath = System.getProperty("vespa.test.config"); - return configPath != null ? fromFile(configPath) : fromController(); + if (configPath != null) { + System.out.println("TestRuntime: Using test config from " + configPath); + return fromFile(configPath); + } + else { + System.out.println("TestRuntime: Using test config from Vespa Cloud"); + return fromController(); + } } private static TestConfig fromFile(String path) { @@ -76,6 +83,8 @@ public class VespaTestRuntime implements TestRuntime { Environment environment = Properties.environment().orElse(Environment.dev); ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) .orElseGet(() -> controller.defaultZone(environment)); + System.out.println("TestRuntime: Requesting endpoint config for tenant.application.instance: " + id.toFullString()); + System.out.println("TestRuntime: Zone: " + zone.toString()); return controller.testConfig(id, zone); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java index e453aaf5bdb..ea4b6a2b02d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java @@ -38,4 +38,7 @@ public interface ConfigChangeAction { /** Returns whether this change should be allowed */ boolean allowed(); + /** Returns whether this change should be ignored for internal redeploy */ + boolean ignoreForInternalRedeploy(); + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 756961933db..0c78aafcf20 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -69,10 +69,10 @@ public interface ModelContext { default int defaultNumResponseThreads() { return 2; } - // TODO Revisit in May or June 2020 + // TODO(bjorncs) Temporary feature flag double threadPoolSizeFactor(); - // TODO Revisit in May or June 2020 + // TODO(bjorncs) Temporary feature flag double queueSizeFactor(); /// Default setting for the gc-options attribute if not specified explicit by application @@ -84,6 +84,9 @@ public interface ModelContext { boolean skipCommunicationManagerThread(); boolean skipMbusRequestThread(); boolean skipMbusReplyThread(); + boolean tlsUseFSync(); + String tlsCompressionType(); + double visibilityDelay(); boolean useContentNodeBtreeDb(); @@ -92,7 +95,9 @@ public interface ModelContext { // TODO Remove on 7.XXX when this is default on. boolean useDirectStorageApiRpc(); - default String proxyProtocol() { return "https+proxy-protocol"; } // TODO bjorncs: Remove after end of May + // TODO(bjorncs) Temporary feature flag + default String proxyProtocol() { return "https+proxy-protocol"; } + default Optional<AthenzDomain> athenzDomain() { return Optional.empty(); } // TODO(mpolden): Remove after May 2020 @@ -110,6 +115,9 @@ public interface ModelContext { return Quota.empty(); } + // TODO(bjorncs): Temporary feature flag + default boolean useNewRestapiHandler() { return false; } + } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 22bdf31350a..31c72e1be69 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -38,9 +38,12 @@ public class TestProperties implements ModelContext.Properties { private boolean useContentNodeBtreeDb = false; private boolean useThreePhaseUpdates = false; private boolean useDirectStorageApiRpc = false; + private boolean tlsUseFSync = false; + private String tlsCompressionType = "NONE"; private double defaultTermwiseLimit = 1.0; private double threadPoolSizeFactor = 0.0; private double queueSizeFactor = 0.0; + private double visibilityDelay = 0.0; private String jvmGCOptions = null; private String sequencerType = "LATENCY"; private String responseSequencerType = "ADAPTIVE"; @@ -83,6 +86,9 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean skipMbusRequestThread() { return false; } @Override public boolean skipMbusReplyThread() { return false; } @Override public Quota quota() { return quota; } + @Override public double visibilityDelay() { return visibilityDelay; } + @Override public boolean tlsUseFSync() { return tlsUseFSync; } + @Override public String tlsCompressionType() { return tlsCompressionType; } public TestProperties setJvmGCOptions(String gcOptions) { jvmGCOptions = gcOptions; @@ -139,6 +145,21 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setTlsUseFSync(boolean useFSync) { + this.tlsUseFSync = useFSync; + return this; + } + + public TestProperties setTlsCompressionType(String type) { + this.tlsCompressionType = type; + return this; + } + + public TestProperties setVisibilityDelay(double visibilityDelay) { + this.visibilityDelay = visibilityDelay; + return this; + } + public TestProperties setMultitenant(boolean multitenant) { this.multitenant = multitenant; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 120e323e652..56b70bec24e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.monitoring; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; @@ -129,7 +130,12 @@ public class VespaMetricSet { metrics.add(new Metric("serverActiveThreads.count")); metrics.add(new Metric("serverActiveThreads.last")); - metrics.add(new Metric("jdisc.thread_pool.unhandled_exceptions.rate")); + { + List<String> suffices = List.of("sum", "count", "last", "min", "max"); + addMetric(metrics, "jdisc.thread_pool.unhandled_exceptions", suffices); + addMetric(metrics, "jdisc.thread_pool.work_queue.capacity", suffices); + addMetric(metrics, "jdisc.thread_pool.work_queue.size", suffices); + } metrics.add(new Metric("httpapi_latency.max")); metrics.add(new Metric("httpapi_latency.sum")); @@ -399,6 +405,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); + metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 @@ -687,4 +694,10 @@ public class VespaMetricSet { return metrics; } + private static void addMetric(Set<Metric> metrics, String metricName, List<String> aggregateSuffices) { + for (String suffix : aggregateSuffices) { + metrics.add(new Metric(metricName + "." + suffix)); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index f6108f5ac77..c0a2fe4fdf2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; -import java.util.Optional; +import java.util.Locale; import java.util.stream.Collectors; /** @@ -23,16 +23,15 @@ public class QuotaValidator extends Validator { } private void validateBudget(int budget, VespaModel model) { - Optional<Double> spend = model.allClusters().stream() + var spend = model.allClusters().stream() .map(clusterId -> model.provisioned().all().get(clusterId)) .map(Capacity::maxResources) .map(clusterCapacity -> clusterCapacity.nodeResources().cost() * clusterCapacity.nodes()) - .reduce(Double::sum); + .reduce(0.0, Double::sum); - if(spend.isPresent() && spend.get() > budget) - throw new IllegalArgumentException( - String.format("Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%d)!", - spend.get(), budget)); + if (spend > budget) { + throwBudgetExceeded(spend, budget); + } } /** Check that all clusters in the application do not exceed the quota max cluster size. */ @@ -51,4 +50,9 @@ public class QuotaValidator extends Validator { throw new IllegalArgumentException("Clusters " + clusterNames + " exceeded max cluster size of " + maxClusterSize); } } + + private void throwBudgetExceeded(double spend, double budget) { + var message = String.format(Locale.US, "Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%.2f)!", spend, budget); + throw new IllegalArgumentException(message); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java index 5cee08ea9af..3176ad9f912 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java @@ -31,7 +31,7 @@ public class ContainerRestartValidator implements ChangeValidator { } private static ConfigChangeAction createConfigChangeAction(Container container) { - return new VespaRestartAction(createMessage(container), container.getServiceInfo()); + return new VespaRestartAction(createMessage(container), container.getServiceInfo(), true); } private static String createMessage(Container container) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java index 0fd38e5dbdd..be43c6eddfb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.application.api.ValidationOverrides; import java.time.Instant; -import java.util.Collections; import java.util.List; /** @@ -27,31 +26,29 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config private final String documentType; private final boolean allowed; - private final Instant now; - private VespaRefeedAction(String name, String message, List<ServiceInfo> services, String documentType, boolean allowed, Instant now) { + private VespaRefeedAction(String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { super(message, services); this.name = name; this.documentType = documentType; this.allowed = allowed; - this.now = now; } /** Creates a refeed action with some missing information */ // TODO: We should require document type or model its absence properly public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, Instant now) { - return new VespaRefeedAction(name, message, Collections.emptyList(), "", overrides.allows(name, now), now); + return new VespaRefeedAction(name, message, List.of(), "", overrides.allows(name, now)); } /** Creates a refeed action */ public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, List<ServiceInfo> services, String documentType, Instant now) { - return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name, now), now); + return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name, now)); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed, now); + return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed); } @Override @@ -64,6 +61,11 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config public boolean allowed() { return allowed; } @Override + public boolean ignoreForInternalRedeploy() { + return false; + } + + @Override public String toString() { return super.toString() + ", documentType='" + documentType + "'"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java index d42db15d062..3ea18cac1d6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeRestartAction; import com.yahoo.config.model.api.ServiceInfo; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -15,16 +13,24 @@ import java.util.List; */ public class VespaRestartAction extends VespaConfigChangeAction implements ConfigChangeRestartAction { + private final boolean ignoreForInternalRedeploy; + public VespaRestartAction(String message) { - super(message, new ArrayList<>()); + this(message, List.of()); } public VespaRestartAction(String message, ServiceInfo service) { - super(message, Collections.singletonList(service)); + this(message, List.of(service)); + } + + public VespaRestartAction(String message, ServiceInfo services, boolean ignoreForInternalRedeploy) { + super(message, List.of(services)); + this.ignoreForInternalRedeploy = ignoreForInternalRedeploy; } public VespaRestartAction(String message, List<ServiceInfo> services) { super(message, services); + this.ignoreForInternalRedeploy = false; } @Override @@ -32,4 +38,25 @@ public class VespaRestartAction extends VespaConfigChangeAction implements Confi return new VespaRestartAction(newMessage, newServices); } + @Override + public boolean ignoreForInternalRedeploy() { + return ignoreForInternalRedeploy; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + + VespaRestartAction that = (VespaRestartAction) o; + return ignoreForInternalRedeploy == that.ignoreForInternalRedeploy; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (ignoreForInternalRedeploy ? 1 : 0); + return result; + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index feaa6eb5940..9018a0231db 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -41,13 +41,17 @@ public class ContainerDocumentApi { private static void addRestApiHandler(ContainerCluster<?> cluster, Options options) { - var handler = newVespaClientHandler( - "com.yahoo.document.restapi.resource.RestApi", "/document/v1/*", options); + String handlerClassName = options.useNewRestapiHandler + ? "com.yahoo.document.restapi.resource.DocumentV1ApiHandler" + : "com.yahoo.document.restapi.resource.RestApi"; + var handler = newVespaClientHandler(handlerClassName, "/document/v1/*", options); cluster.addComponent(handler); - var executor = new Threadpool( - "restapi-handler", cluster, options.restApiThreadpoolOptions, options.feedThreadPoolSizeFactor); - handler.inject(executor); - handler.addComponent(executor); + if (!options.useNewRestapiHandler) { + var executor = new Threadpool( + "restapi-handler", cluster, options.restApiThreadpoolOptions, options.feedThreadPoolSizeFactor); + handler.inject(executor); + handler.addComponent(executor); + } } private static Handler<AbstractConfigProducer<?>> newVespaClientHandler( @@ -76,15 +80,18 @@ public class ContainerDocumentApi { private final ContainerThreadpool.UserOptions restApiThreadpoolOptions; private final ContainerThreadpool.UserOptions feedApiThreadpoolOptions; private final double feedThreadPoolSizeFactor; + private final boolean useNewRestapiHandler; public Options(Collection<String> bindings, ContainerThreadpool.UserOptions restApiThreadpoolOptions, ContainerThreadpool.UserOptions feedApiThreadpoolOptions, - double feedThreadPoolSizeFactor) { + double feedThreadPoolSizeFactor, + boolean useNewRestapiHandler) { this.bindings = Collections.unmodifiableCollection(bindings); this.restApiThreadpoolOptions = restApiThreadpoolOptions; this.feedApiThreadpoolOptions = feedApiThreadpoolOptions; this.feedThreadPoolSizeFactor = feedThreadPoolSizeFactor; + this.useNewRestapiHandler = useNewRestapiHandler; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 1427fa492dc..7822b03db08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -94,6 +94,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider"); + addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName()); addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index 99ae6184f5c..3baf792dfba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -25,7 +25,8 @@ public class DocumentApiOptionsBuilder { getBindings(spec), threadpoolOptions(spec, "rest-api"), threadpoolOptions(spec, "http-client-api"), - deployState.getProperties().feedCoreThreadPoolSizeFactor()); + deployState.getProperties().feedCoreThreadPoolSizeFactor(), + deployState.getProperties().useNewRestapiHandler()); } private static ContainerThreadpool.UserOptions threadpoolOptions(Element spec, String elementName) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index f329f62ee7b..2198b6e278b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -200,6 +200,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot this.flushOnShutdown = flushOnShutdown; this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); + visibilityDelay = featureFlags.visibilityDelay(); } public void setVisibilityDelay(double delay) { @@ -267,7 +268,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState.getDeployLogger()); - tls = new TransactionLogServer(searchNode, clusterName); + tls = new TransactionLogServer(searchNode, clusterName, deployState.getProperties()); tls.setHostResource(searchNode.getHostResource()); tls.initService(deployState.getDeployLogger()); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java index 2c457940f24..347250176e5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -15,12 +16,24 @@ import org.w3c.dom.Element; public class TransactionLogServer extends AbstractService { private static final long serialVersionUID = 1L; + private final boolean useFSync; + private final TranslogserverConfig.Compression.Type.Enum compressionType; - public TransactionLogServer(AbstractConfigProducer searchNode, String clusterName) { + private static TranslogserverConfig.Compression.Type.Enum convertCompressionType(String type) { + try { + return TranslogserverConfig.Compression.Type.Enum.valueOf(type); + } catch (Throwable t) { + return TranslogserverConfig.Compression.Type.NONE; + } + } + + public TransactionLogServer(AbstractConfigProducer searchNode, String clusterName, ModelContext.Properties featureFlags) { super(searchNode, "transactionlogserver"); portsMeta.on(0).tag("tls"); setProp("clustername", clusterName); setProp("clustertype", "search"); + useFSync = featureFlags.tlsUseFSync(); + compressionType = convertCompressionType(featureFlags.tlsCompressionType()); } public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<TransactionLogServer> { @@ -31,7 +44,7 @@ public class TransactionLogServer extends AbstractService { @Override protected TransactionLogServer doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { - return new TransactionLogServer(ancestor, clusterName); + return new TransactionLogServer(ancestor, clusterName, deployState.getProperties()); } } @@ -65,6 +78,8 @@ public class TransactionLogServer extends AbstractService { public void getConfig(TranslogserverConfig.Builder builder) { builder.listenport(getTlsPort()).basedir(getTlsDir()); + builder.usefsync(useFSync); + builder.compression.type(compressionType); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 0a1c0dcb8d8..10199bfe6b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -42,7 +42,7 @@ public class QuotaValidatorTest { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); } catch (RuntimeException e) { - assertEquals("Hourly spend for maximum specified resources ($1.60) exceeds budget from quota ($1)!", e.getMessage()); + assertEquals("Hourly spend for maximum specified resources ($1.60) exceeds budget from quota ($1.00)!", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 99f8b8cbb5e..10049daa541 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -229,7 +229,6 @@ public class ContentBuilderTest extends DomBuilderTest { /* Not yet assertNotNull(h.getService("qrserver")); - assertNotNull(h.getService("topleveldisptach")); assertNotNull(h.getService("docproc")); */ @@ -760,15 +759,12 @@ public class ContentBuilderTest extends DomBuilderTest { } } - private void verifyFeedSequencer(String input, String expected) { - verifyFeedSequencer(input, expected, 0); - } - private void verifyFeedSequencer(String input, String expected, double visibilityDelay) { - String hostedXml = "<services>" + + private String xmlWithVisibilityDelay(Double visibilityDelay) { + return "<services>" + "<content version='1.0' id='search'>" + " <redundancy>1</redundancy>" + " <search>" + - " <visibility-delay>" + visibilityDelay + "</visibility-delay>" + + ((visibilityDelay != null) ? " <visibility-delay>" + visibilityDelay + "</visibility-delay>" : "") + " </search>" + " <documents>" + " <document type='music' mode='index'/>" + @@ -776,6 +772,13 @@ public class ContentBuilderTest extends DomBuilderTest { " <nodes count='1'/>" + "</content>" + "</services>"; + } + + private void verifyFeedSequencer(String input, String expected) { + verifyFeedSequencer(input, expected, 0); + } + private void verifyFeedSequencer(String input, String expected, double visibilityDelay) { + String hostedXml = xmlWithVisibilityDelay(visibilityDelay); DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setFeedSequencerType(input)); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() @@ -800,6 +803,25 @@ public class ContentBuilderTest extends DomBuilderTest { } + private void verifyThatFeatureFlagControlsVisibilityDelayDefault(double defaultVisibiliDelay, Double xmlOverride, double expected) { + String hostedXml = xmlWithVisibilityDelay(xmlOverride); + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setVisibilityDelay(defaultVisibiliDelay)); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(hostedXml) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(expected, config.documentdb(0).visibilitydelay(), 0.0); + } + @Test + public void verifyThatFeatureFlagControlsVisibilityDelayDefault() { + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.0, null, 0.0); + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.3, null, 0.3); + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.0, 0.5, 0.5); + verifyThatFeatureFlagControlsVisibilityDelayDefault(0.3, 0.6, 0.6); + } + @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index 504c3d9ba9c..5d4756c430d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -143,7 +143,7 @@ public class IndexedTest extends ContentBaseTest { // HostResource h = model.getHostSystem().getHosts().get(0); // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", - // "qrserver", "fleetcontroller", "topleveldispatch", + // "qrserver", "fleetcontroller", // "storagenode", "searchnode", "distributor", "transactionlogserver"}; // DomContentBuilderTest.assertServices(h, expectedServices); Routing routing = model.getRouting(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index 9c69ba8f212..a04a4f196cf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -1,8 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search.test; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.Host; @@ -36,7 +40,7 @@ public class SearchNodeTest { private void prepare(MockRoot root, SearchNode node) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster"); + TransactionLogServer tls = new TransactionLogServer(root, "mycluster", root.getDeployState().getProperties()); tls.setHostResource(new HostResource(host)); tls.setBasePort(100); tls.initService(root.deployLogger()); @@ -52,6 +56,10 @@ public class SearchNodeTest { return SearchNode.create(parent, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, combined); } + private static SearchNode createSearchNode(MockRoot root) { + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, false); + } + @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); @@ -80,4 +88,34 @@ public class SearchNodeTest { CoreMatchers.containsString("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); } + private MockRoot createRoot(ModelContext.Properties properties) { + return new MockRoot("", new DeployState.Builder().properties(properties).build()); + } + + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties) { + MockRoot root = createRoot(properties); + SearchNode node = createSearchNode(root); + prepare(root, node); + TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); + node.getConfig(tlsBuilder); + return tlsBuilder.build(); + } + + @Test + public void requireThaFeatureFlagCanControlTlsUseFSync() { + assertFalse(getTlsConfig(new TestProperties()).usefsync()); + assertFalse(getTlsConfig(new TestProperties().setTlsUseFSync(false)).usefsync()); + assertTrue(getTlsConfig(new TestProperties().setTlsUseFSync(true)).usefsync()); + } + + @Test + public void requireThaFeatureFlagCanControlCompressionType() { + assertEquals(TranslogserverConfig.Compression.Type.NONE, getTlsConfig(new TestProperties()).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.NONE, getTlsConfig(new TestProperties().setTlsCompressionType("NONE")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.NONE_MULTI, getTlsConfig(new TestProperties().setTlsCompressionType("NONE_MULTI")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.ZSTD, getTlsConfig(new TestProperties().setTlsCompressionType("ZSTD")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.LZ4, getTlsConfig(new TestProperties().setTlsCompressionType("LZ4")).compression().type()); + assertEquals(TranslogserverConfig.Compression.Type.NONE, getTlsConfig(new TestProperties().setTlsCompressionType("zstd")).compression().type()); + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 9cae0a08360..1c53ff18222 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -1,6 +1,9 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import java.text.DecimalFormat; +import java.text.FieldPosition; +import java.text.NumberFormat; import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -215,13 +218,33 @@ public class NodeResources { return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } + private static StringBuffer appendDouble(StringBuffer sb, double d) { + long x10 = Math.round(d*10); + sb.append(x10/10).append('.').append(x10%10); + return sb; + } @Override public String toString() { - return String.format(Locale.ENGLISH, "[vcpu: %1$.1f, memory: %2$.1f Gb, disk %3$.1f Gb" + - (bandwidthGbps > 0 ? ", bandwidth: %4$.1f Gbps" : "") + - ( ! diskSpeed.isDefault() ? ", disk speed: " + diskSpeed : "") + - ( ! storageType.isDefault() ? ", storage type: " + storageType : "") + "]", - vcpu, memoryGb, diskGb, bandwidthGbps); + StringBuffer sb = new StringBuffer("[vcpu: "); + appendDouble(sb, vcpu); + sb.append(", memory: "); + appendDouble(sb, memoryGb); + sb.append(" Gb, disk "); + appendDouble(sb, diskGb); + sb.append(" Gb"); + if (bandwidthGbps > 0) { + sb.append(", bandwidth: "); + appendDouble(sb, bandwidthGbps); + sb.append(" Gbps"); + } + if ( !diskSpeed.isDefault()) { + sb.append(", disk speed: ").append(diskSpeed); + } + if ( !storageType.isDefault()) { + sb.append(", storage type: ").append(storageType); + } + sb.append(']'); + return sb.toString(); } /** Returns true if all the resources of this are the same or larger than the given resources */ diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java index 21e1afeed17..044afa72a5d 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java @@ -16,6 +16,32 @@ public class NodeResourcesTest { new NodeResources(1., 10., 100., 0).toString()); assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk 33.3 Gb, bandwidth: 0.3 Gbps]", new NodeResources(1/3., 10/3., 100/3., 0.3).toString()); + assertEquals("[vcpu: 0.7, memory: 9.0 Gb, disk 66.7 Gb, bandwidth: 0.7 Gbps]", + new NodeResources(2/3., 8.97, 200/3., 0.67).toString()); + } + + private long runTest(NodeResources [] resouces, int num) { + long sum = 0; + for (int i = 0; i < num; i++) { + for (NodeResources ns :resouces) { + sum += ns.toString().length(); + } + } + return sum; + } + @Test + public void benchmark() { + NodeResources [] resouces = new NodeResources[100]; + for (int i = 0; i < resouces.length; i++) { + resouces[i] = new NodeResources(1/3., 10/3., 100/3., 0.3); + } + int NUM_ITER = 100; // Use at least 100000 for proper benchmarking + long warmup = runTest(resouces, NUM_ITER); + long start = System.nanoTime(); + long benchmark = runTest(resouces, NUM_ITER); + long duration = System.nanoTime() - start; + System.out.println("NodeResources.toString() took " + duration/1000000 + " ms"); + assertEquals(warmup, benchmark); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 34510805aca..4635ab4fd86 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -8,12 +8,19 @@ import com.yahoo.jrt.Transport; import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.proxy.filedistribution.FileDistributionAndUrlDownload; import com.yahoo.yolean.system.CatchSignals; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; @@ -28,6 +35,7 @@ import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT; */ public class ProxyServer implements Runnable { + private static final DaemonThreadFactory threadFactory = new DaemonThreadFactory("ProxyServer"); private static final int DEFAULT_RPC_PORT = 19090; private static final int JRT_TRANSPORT_THREADS = 4; static final String DEFAULT_PROXY_CONFIG_SOURCES = "tcp/localhost:19070"; @@ -58,9 +66,8 @@ public class ProxyServer implements Runnable { @Override public void run() { if (rpcServer != null) { - Thread t = new Thread(rpcServer); + Thread t = threadFactory.newThread(rpcServer); t.setName("RpcServer"); - t.setDaemon(true); t.start(); } } @@ -125,7 +132,15 @@ public class ProxyServer implements Runnable { } } } - stop(); + ExecutorService executor = Executors.newSingleThreadExecutor(threadFactory); + Callable<String> stopper = () -> { stop(); return "clean shutdown"; }; + Future<String> future = executor.submit(stopper); + try { + String result = future.get(5, TimeUnit.SECONDS); + Event.stopping("configproxy", result); + } catch (Exception e) { + System.exit(1); + } System.exit(0); } @@ -146,9 +161,8 @@ public class ProxyServer implements Runnable { ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, new MemoryCache(), null); // catch termination and interrupt signal proxyServer.setupSignalHandler(); - Thread proxyserverThread = new Thread(proxyServer); + Thread proxyserverThread = threadFactory.newThread(proxyServer); proxyserverThread.setName("configproxy"); - proxyserverThread.setDaemon(true); proxyserverThread.start(); proxyServer.waitForShutdown(); } @@ -174,11 +188,15 @@ public class ProxyServer implements Runnable { } void stop() { - Event.stopping("configproxy", "shutdown"); + Event.stopping("configproxy", "shutdown rpcServer"); if (rpcServer != null) rpcServer.shutdown(); + Event.stopping("configproxy", "cancel configClient"); if (configClient != null) configClient.cancel(); + Event.stopping("configproxy", "flush"); flush(); + Event.stopping("configproxy", "close fileDistribution"); fileDistributionAndUrlDownload.close(); + Event.stopping("configproxy", "stop complete"); } MemoryCache getMemoryCache() { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index dfd81341e5d..095bde76c39 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -182,9 +182,6 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { shutdownSourceConnections(); delayedResponsesFuture.cancel(true); delayedResponsesScheduler.shutdownNow(); - nextConfigFuture.cancel(true); - nextConfigScheduler.shutdownNow(); - requester.close(); supervisor.transport().shutdown().join(); } @@ -195,6 +192,7 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { public void shutdownSourceConnections() { activeSubscribers.values().forEach(Subscriber::cancel); activeSubscribers.clear(); + nextConfigFuture.cancel(true); nextConfigScheduler.shutdownNow(); requester.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3564a6e6da7..4670184c303 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -24,7 +24,6 @@ import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.path.Path; -import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.application.Application; @@ -65,7 +64,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -128,6 +130,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final LogRetriever logRetriever; private final TesterClient testerClient; private final Metric metric; + private final BooleanFlag deployWithInternalRestart; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -177,6 +180,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.clock = Objects.requireNonNull(clock); this.testerClient = Objects.requireNonNull(testerClient); this.metric = Objects.requireNonNull(metric); + this.deployWithInternalRestart = Flags.DEPLOY_WITH_INTERNAL_RESTART.bindTo(flagSource); } public static class Builder { @@ -280,28 +284,29 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye bootstrapping.set(false); } - public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, Instant now) { + public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams) { + DeployHandlerLogger logger = DeployHandlerLogger.forPrepareParams(prepareParams); + Deployment deployment = prepare(tenant, sessionId, prepareParams, logger); + return new PrepareResult(sessionId, deployment.configChangeActions(), logger); + } + + private Deployment prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, DeployHandlerLogger logger) { validateThatLocalSessionIsNotActive(tenant, sessionId); LocalSession session = getLocalSession(tenant, sessionId); ApplicationId applicationId = prepareParams.getApplicationId(); - Optional<ApplicationSet> currentActiveApplicationSet = getCurrentActiveApplicationSet(tenant, applicationId); - Slime deployLog = createDeployLog(); - DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); - try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { - SessionRepository sessionRepository = tenant.getSessionRepository(); - ConfigChangeActions actions = sessionRepository.prepareLocalSession(session, logger, prepareParams, - currentActiveApplicationSet, tenant.getPath(), now); - logConfigChangeActions(actions, logger); - log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); - return new PrepareResult(sessionId, actions, deployLog); - } + Deployment deployment = Deployment.unprepared(session, this, hostProvisioner, tenant, prepareParams, logger, clock); + deployment.prepare(); + + logConfigChangeActions(deployment.configChangeActions(), logger); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); + return deployment; } - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, Instant now) { + public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) { File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); PrepareResult prepareResult; try { - prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, now); + prepareResult = deploy(decompressApplication(in, tempDir), prepareParams); } finally { cleanupTempDirectory(tempDir); } @@ -309,16 +314,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { - return deploy(applicationPackage, prepareParams, Instant.now()); - } - - public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams, Instant now) { ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = getTenant(applicationId); - PrepareResult result = prepare(tenant, sessionId, prepareParams, now); - activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); - return result; + DeployHandlerLogger logger = DeployHandlerLogger.forPrepareParams(prepareParams); + Deployment deployment = prepare(tenant, sessionId, prepareParams, logger); + deployment.activate(); + + return new PrepareResult(sessionId, deployment.configChangeActions(), logger); } /** @@ -379,9 +382,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye SessionRepository sessionRepository = tenant.getSessionRepository(); LocalSession newSession = sessionRepository.createSessionFromExisting(activeSession, logger, true, timeoutBudget); sessionRepository.addLocalSession(newSession); + boolean internalRestart = deployWithInternalRestart.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); - return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, - false /* don't validate as this is already deployed */, bootstrap)); + return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, logger, timeout, clock, + false /* don't validate as this is already deployed */, bootstrap, internalRestart)); } @Override @@ -398,15 +402,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye TimeoutBudget timeoutBudget, boolean force) { LocalSession localSession = getLocalSession(tenant, sessionId); - Deployment deployment = deployment(localSession, tenant, timeoutBudget.timeLeft(), force); + Deployment deployment = Deployment.prepared(localSession, this, hostProvisioner, tenant, logger, timeoutBudget.timeout(), clock, false, force); deployment.activate(); return localSession.getApplicationId(); } - private Deployment deployment(LocalSession session, Tenant tenant, Duration timeout, boolean force) { - return Deployment.prepared(session, this, hostProvisioner, tenant, timeout, clock, false, force); - } - public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean force) { Tenant tenant = tenantRepository.getTenant(prepared.getTenantName()); Transaction transaction = tenant.getSessionRepository().createActivateTransaction(prepared); @@ -520,8 +520,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye transaction.add(new ApplicationRolesStore(curator, tenant.getPath()).delete(applicationId)); // Delete endpoint certificates transaction.add(new EndpointCertificateMetadataStore(curator, tenant.getPath()).delete(applicationId)); - // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server - // this is done asynchronously in application maintenance by the node repository) + // This call will remove application in zookeeper. Watches in TenantApplications will remove the application + // and allocated hosts in model and handlers in RPC server transaction.add(tenantApplications.createDeleteTransaction(applicationId)); hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); @@ -766,7 +766,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @return the active session, or null if there is no active session for the given application id. */ public RemoteSession getActiveSession(ApplicationId applicationId) { - return getActiveSession(getTenant(applicationId), applicationId); + Tenant tenant = getTenant(applicationId); + if (tenant == null) throw new IllegalArgumentException("Could not find any tenant for '" + applicationId + "'"); + return getActiveSession(tenant, applicationId); } public long getSessionIdForApplication(ApplicationId applicationId) { @@ -1057,12 +1059,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .getPort(); } - public Slime createDeployLog() { - Slime deployLog = new Slime(); - deployLog.setObject(); - return deployLog; - } - public Zone zone() { return new Zone(SystemName.from(configserverConfig.system()), Environment.from(configserverConfig.environment()), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 609ff4473c6..3275dc42477 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -57,7 +57,6 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final ApplicationRepository applicationRepository; private final RpcServer server; - private final Optional<Thread> serverThread; private final VersionState versionState; private final StateMonitor stateMonitor; private final VipStatus vipStatus; @@ -66,6 +65,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; private final ExecutorService rpcServerExecutor; + private final Optional<ExecutorService> bootstrapExecutor; @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, @@ -96,20 +96,21 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); + log.log(Level.FINE, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); initializing(vipStatusMode); switch (mode) { case BOOTSTRAP_IN_SEPARATE_THREAD: - this.serverThread = Optional.of(new Thread(this, "config server bootstrap thread")); - serverThread.get().start(); + bootstrapExecutor = Optional.of(Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server bootstrap"))); + bootstrapExecutor.get().execute(this); break; case BOOTSTRAP_IN_CONSTRUCTOR: - this.serverThread = Optional.empty(); + bootstrapExecutor = Optional.empty(); start(); break; case INITIALIZE_ONLY: - this.serverThread = Optional.empty(); + bootstrapExecutor = Optional.empty(); break; default: throw new IllegalArgumentException("Unknown bootstrap mode " + mode + ", legal values: " + Arrays.toString(Mode.values())); @@ -123,13 +124,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable server.stop(); log.log(Level.FINE, "RPC server stopped"); rpcServerExecutor.shutdown(); - serverThread.ifPresent(thread -> { - try { - thread.join(); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); - } - }); + bootstrapExecutor.ifPresent(ExecutorService::shutdownNow); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index 012d0d52275..ab1bd79d498 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -147,6 +147,11 @@ public class FileDistributionStatus extends AbstractComponent { } } + @Override + public void deconstruct() { + rpcExecutor.shutdownNow(); + } + static class HostStatus { private final String hostname; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java index d783dc105c3..dd9c8e4b6bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.configchange; import com.yahoo.config.model.api.ConfigChangeAction; import java.util.List; +import java.util.Objects; /** * Contains an aggregated view of which actions that must be performed to handle config @@ -18,13 +19,16 @@ public class ConfigChangeActions { private final RefeedActions refeedActions; public ConfigChangeActions() { - this.restartActions = new RestartActions(); - this.refeedActions = new RefeedActions(); + this(new RestartActions(), new RefeedActions()); } public ConfigChangeActions(List<ConfigChangeAction> actions) { - this.restartActions = new RestartActions(actions); - this.refeedActions = new RefeedActions(actions); + this(new RestartActions(actions), new RefeedActions(actions)); + } + + public ConfigChangeActions(RestartActions restartActions, RefeedActions refeedActions) { + this.restartActions = Objects.requireNonNull(restartActions); + this.refeedActions = Objects.requireNonNull(refeedActions); } public RestartActions getRestartActions() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java index 29b0b99e42e..fab36246a41 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import java.util.*; +import java.util.stream.Collectors; /** * Represents all actions to restart services in order to handle a config change. @@ -18,6 +19,7 @@ public class RestartActions { private final String clusterName; private final String clusterType; private final String serviceType; + private final boolean ignoreForInternalRedeploy; private final Set<ServiceInfo> services = new LinkedHashSet<>(); private final Set<String> messages = new TreeSet<>(); @@ -31,10 +33,11 @@ public class RestartActions { return this; } - private Entry(String clusterName, String clusterType, String serviceType) { + private Entry(String clusterName, String clusterType, String serviceType, boolean ignoreForInternalRedeploy) { this.clusterName = clusterName; this.clusterType = clusterType; this.serviceType = serviceType; + this.ignoreForInternalRedeploy = ignoreForInternalRedeploy; } public String getClusterName() { @@ -49,6 +52,10 @@ public class RestartActions { return serviceType; } + public boolean ignoreForInternalRedeploy() { + return ignoreForInternalRedeploy; + } + public Set<ServiceInfo> getServices() { return services; } @@ -59,28 +66,19 @@ public class RestartActions { } - private Entry addEntry(ServiceInfo service) { - String clusterName = service.getProperty("clustername").orElse(""); - String clusterType = service.getProperty("clustertype").orElse(""); - String entryId = clusterType + "." + clusterName + "." + service.getServiceType(); - Entry entry = actions.get(entryId); - if (entry == null) { - entry = new Entry(clusterName, clusterType, service.getServiceType()); - actions.put(entryId, entry); - } - return entry; - } - private final Map<String, Entry> actions = new TreeMap<>(); - public RestartActions() { + public RestartActions() { } + + private RestartActions(Map<String, Entry> actions) { + this.actions.putAll(actions); } public RestartActions(List<ConfigChangeAction> actions) { for (ConfigChangeAction action : actions) { if (action.getType().equals(ConfigChangeAction.Type.RESTART)) { for (ServiceInfo service : action.getServices()) { - addEntry(service). + addEntry(service, action.ignoreForInternalRedeploy()). addService(service). addMessage(action.getMessage()); } @@ -88,6 +86,24 @@ public class RestartActions { } } + private Entry addEntry(ServiceInfo service, boolean ignoreForInternalRedeploy) { + String clusterName = service.getProperty("clustername").orElse(""); + String clusterType = service.getProperty("clustertype").orElse(""); + String entryId = clusterType + "." + clusterName + "." + service.getServiceType() + "." + ignoreForInternalRedeploy; + Entry entry = actions.get(entryId); + if (entry == null) { + entry = new Entry(clusterName, clusterType, service.getServiceType(), ignoreForInternalRedeploy); + actions.put(entryId, entry); + } + return entry; + } + + public RestartActions useForInternalRestart(boolean useForInternalRestart) { + return new RestartActions(actions.entrySet().stream() + .filter(entry -> !useForInternalRestart || !entry.getValue().ignoreForInternalRedeploy()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + public List<Entry> getEntries() { return new ArrayList<>(actions.values()); } @@ -97,6 +113,6 @@ public class RestartActions { } public boolean isEmpty() { - return getEntries().isEmpty(); + return actions.isEmpty(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index c33c5ff9f57..110c6464eba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -4,9 +4,11 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.util.logging.Level; @@ -22,25 +24,24 @@ public class DeployHandlerLogger implements DeployLogger { private static final Logger log = Logger.getLogger(DeployHandlerLogger.class.getName()); - private final Cursor logroot; + private final String prefix; private final boolean verbose; - private final ApplicationId app; + private final Slime slime; + private final Cursor logroot; - public DeployHandlerLogger(Cursor root, boolean verbose, ApplicationId app) { - logroot = root; + private DeployHandlerLogger(String prefix, boolean verbose) { + this.prefix = prefix; this.verbose = verbose; - this.app = app; + this.slime = new Slime(); + this.logroot = slime.setObject().setArray("log"); } @Override public void log(Level level, String message) { - if ((level == Level.FINE || - level == LogLevel.DEBUG || - level == LogLevel.SPAM) && - !verbose) { + if ((level == Level.FINE || level == LogLevel.DEBUG || level == LogLevel.SPAM) && !verbose) return; - } - String fullMsg = TenantRepository.logPre(app) + message; + + String fullMsg = prefix + message; Cursor entry = logroot.addObject(); entry.setLong("time", System.currentTimeMillis()); entry.setString("level", level.getName()); @@ -49,4 +50,19 @@ public class DeployHandlerLogger implements DeployLogger { log.log(Level.FINE, fullMsg); } + public Slime slime() { + return slime; + } + + public static DeployHandlerLogger forApplication(ApplicationId app, boolean verbose) { + return new DeployHandlerLogger(TenantRepository.logPre(app), verbose); + } + + public static DeployHandlerLogger forTenant(TenantName tenantName, boolean verbose) { + return new DeployHandlerLogger(TenantRepository.logPre(tenantName), verbose); + } + + public static DeployHandlerLogger forPrepareParams(PrepareParams prepareParams) { + return forApplication(prepareParams.getApplicationId(), prepareParams.isVerbose()); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 6c111ff0131..3726ea97fcc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -1,31 +1,34 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.yahoo.component.Version; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; -import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.curator.Lock; import java.time.Clock; import java.time.Duration; import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; @@ -44,82 +47,62 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** The session containing the application instance to activate */ private final LocalSession session; private final ApplicationRepository applicationRepository; - private final Optional<Provisioner> hostProvisioner; + private final Supplier<PrepareParams> params; + private final Optional<Provisioner> provisioner; private final Tenant tenant; - private final Duration timeout; + private final DeployLogger deployLogger; private final Clock clock; - private final DeployLogger logger = new SilentDeployLogger(); - - /** The repository part of docker image this application should run on. Version is separate from image repo */ - final Optional<DockerImage> dockerImageRepository; - - /** The Vespa version this application should run on */ - private final Version version; - - /** True if this deployment is done to bootstrap the config server */ - private final boolean isBootstrap; + private final boolean internalRedeploy; - /** The (optional) Athenz domain this application should use */ - private final Optional<AthenzDomain> athenzDomain; + private boolean prepared; + private ConfigChangeActions configChangeActions; - private boolean prepared = false; - - /** Whether this model should be validated (only takes effect if prepared=false) */ - private final boolean validate; - - /** Whether activation of this model should be forced */ - private final boolean force; - - private Deployment(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, - Clock clock, boolean prepared, boolean validate, boolean isBootstrap, boolean force) { + private Deployment(LocalSession session, ApplicationRepository applicationRepository, Supplier<PrepareParams> params, + Optional<Provisioner> provisioner, Tenant tenant, DeployLogger deployLogger, Clock clock, + boolean internalRedeploy, boolean prepared) { this.session = session; this.applicationRepository = applicationRepository; - this.hostProvisioner = hostProvisioner; + this.params = params; + this.provisioner = provisioner; this.tenant = tenant; - this.timeout = timeout; + this.deployLogger = deployLogger; this.clock = clock; + this.internalRedeploy = internalRedeploy; this.prepared = prepared; - this.validate = validate; - this.dockerImageRepository = session.getDockerImageRepository(); - this.version = session.getVespaVersion(); - this.isBootstrap = isBootstrap; - this.athenzDomain = session.getAthenzDomain(); - this.force = force; } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, - Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false, - validate, isBootstrap, false); + Optional<Provisioner> provisioner, Tenant tenant, PrepareParams params, DeployLogger logger, Clock clock) { + return new Deployment(session, applicationRepository, () -> params, provisioner, tenant, logger, clock, false, false); + } + + public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, + Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger, + Duration timeout, Clock clock, boolean validate, boolean isBootstrap, boolean internalRestart) { + Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, internalRestart); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true, false); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, + Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean isBootstrap, boolean force) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, true, true, isBootstrap, force); + Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, false, force, false); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false, true); } /** Prepares this. This does nothing if this is already prepared */ @Override public void prepare() { if (prepared) return; - ApplicationId applicationId = session.getApplicationId(); - try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + PrepareParams params = this.params.get(); + if (params.internalRestart() && provisioner.isEmpty()) + throw new IllegalArgumentException("Internal restart not supported without Provisioner"); - PrepareParams.Builder params = new PrepareParams.Builder().applicationId(applicationId) - .timeoutBudget(timeoutBudget) - .ignoreValidationErrors(!validate) - .vespaVersion(version.toString()) - .isBootstrap(isBootstrap); - dockerImageRepository.ifPresent(params::dockerImageRepository); - athenzDomain.ifPresent(params::athenzDomain); + ApplicationId applicationId = params.getApplicationId(); + try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { Optional<ApplicationSet> activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); - tenant.getSessionRepository().prepareLocalSession(session, logger, params.build(), activeApplicationSet, - tenant.getPath(), clock.instant()); + this.configChangeActions = tenant.getSessionRepository().prepareLocalSession( + session, deployLogger, params, activeApplicationSet, tenant.getPath(), clock.instant()); this.prepared = true; } } @@ -130,16 +113,17 @@ public class Deployment implements com.yahoo.config.provision.Deployment { prepare(); validateSessionStatus(session); + PrepareParams params = this.params.get(); ApplicationId applicationId = session.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.activateMillis")) { - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + TimeoutBudget timeoutBudget = params.getTimeoutBudget(); if ( ! timeoutBudget.hasTimeLeft()) throw new RuntimeException("Timeout exceeded when trying to activate '" + applicationId + "'"); RemoteSession previousActiveSession; CompletionWaiter waiter; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { previousActiveSession = applicationRepository.getActiveSession(applicationId); - waiter = applicationRepository.activate(session, previousActiveSession, applicationId, force); + waiter = applicationRepository.activate(session, previousActiveSession, applicationId, params.force()); } catch (RuntimeException e) { throw e; @@ -150,10 +134,28 @@ public class Deployment implements com.yahoo.config.provision.Deployment { waiter.awaitCompletion(timeoutBudget.timeLeft()); log.log(Level.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + - hostProvisioner.map(provisioner -> provisioner.getClass().getSimpleName()).orElse("no host provisioner") + + provisioner.map(provisioner -> provisioner.getClass().getSimpleName()).orElse("no host provisioner") + ". Config generation " + session.getMetaData().getGeneration() + (previousActiveSession != null ? ". Based on session " + previousActiveSession.getSessionId() : "") + ". File references: " + applicationRepository.getFileReferences(applicationId)); + + if (params.internalRestart()) { + RestartActions restartActions = configChangeActions.getRestartActions().useForInternalRestart(internalRedeploy); + + if (!restartActions.isEmpty()) { + Set<String> hostnames = restartActions.getEntries().stream() + .flatMap(entry -> entry.getServices().stream()) + .map(ServiceInfo::getHostName) + .collect(Collectors.toUnmodifiableSet()); + + provisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); + deployLogger.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", + hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); + + this.configChangeActions = new ConfigChangeActions(new RestartActions(), configChangeActions.getRefeedActions()); + } + } + return session.getMetaData().getGeneration(); } } @@ -165,12 +167,21 @@ public class Deployment implements com.yahoo.config.provision.Deployment { */ @Override public void restart(HostFilter filter) { - hostProvisioner.get().restart(session.getApplicationId(), filter); + provisioner.get().restart(session.getApplicationId(), filter); } /** Exposes the session of this for testing only */ public LocalSession session() { return session; } + /** + * @return config change actions that need to be performed as result of prepare + * @throws IllegalArgumentException if called without being prepared by this + */ + public ConfigChangeActions configChangeActions() { + if (configChangeActions != null) return configChangeActions; + throw new IllegalArgumentException("No config change actions: " + (prepared ? "was already prepared" : "not yet prepared")); + } + private void validateSessionStatus(LocalSession localSession) { long sessionId = localSession.getSessionId(); if (Session.Status.NEW.equals(localSession.getStatus())) { @@ -180,4 +191,36 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } } + /** + * @param clock system clock + * @param timeout total timeout duration of prepare + activate + * @param session the local session for this deployment + * @param isBootstrap true if this deployment is done to bootstrap the config server + * @param ignoreValidationErrors whether this model should be validated + * @param force whether activation of this model should be forced + */ + private static Supplier<PrepareParams> createPrepareParams( + Clock clock, Duration timeout, LocalSession session, + boolean isBootstrap, boolean ignoreValidationErrors, boolean force, boolean internalRestart) { + + // Supplier because shouldn't/cant create this before validateSessionStatus() for prepared deployments + // memoized because we want to create this once for unprepared deployments + return Suppliers.memoize(() -> { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + + PrepareParams.Builder params = new PrepareParams.Builder() + .applicationId(session.getApplicationId()) + .vespaVersion(session.getVespaVersion().toString()) + .timeoutBudget(timeoutBudget) + .ignoreValidationErrors(ignoreValidationErrors) + .isBootstrap(isBootstrap) + .force(force) + .internalRestart(internalRestart); + session.getDockerImageRepository().ifPresent(params::dockerImageRepository); + session.getAthenzDomain().ifPresent(params::athenzDomain); + + return params.build(); + }); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 87b0ed965d3..48d3fd6a176 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -164,7 +164,11 @@ public class ModelContextImpl implements ModelContext { private final Optional<AthenzDomain> athenzDomain; private final Optional<ApplicationRoles> applicationRoles; private final double feedCoreThreadPoolSizeFactor; + private final double visibilityDelay; private final Quota quota; + private final boolean tlsUseFSync; + private final String tlsCompressionType; + private final boolean useNewRestapiHandler; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -204,6 +208,12 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + visibilityDelay = Flags.VISIBILITY_DELAY.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + tlsCompressionType = Flags.TLS_COMPRESSION_TYPE.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + tlsUseFSync = Flags.TLS_USE_FSYNC.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) @@ -225,6 +235,9 @@ public class ModelContextImpl implements ModelContext { feedCoreThreadPoolSizeFactor = Flags.FEED_CORE_THREAD_POOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.quota = maybeQuota.orElseGet(Quota::empty); + this.useNewRestapiHandler = Flags.USE_NEW_RESTAPI_HANDLER.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()) + .value(); } @Override @@ -313,8 +326,12 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusRequestThread() { return skipMbusRequestThread; } @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public double feedCoreThreadPoolSizeFactor() { return feedCoreThreadPoolSizeFactor; } - + @Override public double visibilityDelay() { return visibilityDelay; } + @Override public boolean tlsUseFSync() { return tlsUseFSync; } + @Override public String tlsCompressionType() { return tlsCompressionType; } @Override public Quota quota() { return quota; } + + @Override public boolean useNewRestapiHandler() { return useNewRestapiHandler; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 99cdb0a74dc..31294faad05 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; @@ -79,8 +80,10 @@ public class FileServer { private FileServer(ConnectionPool connectionPool, File rootDir) { this.downloader = new FileDownloader(connectionPool); this.root = new FileDirectory(rootDir); - this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); - this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); + this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), + new DaemonThreadFactory("file server push")); + this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), + new DaemonThreadFactory("file server pull")); } boolean hasFile(String fileReference) { @@ -191,6 +194,8 @@ public class FileServer { public void close() { downloader.close(); + pullExecutor.shutdown(); + pushExecutor.shutdown(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java index 7112c7d3e23..bbbc8764122 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; @@ -102,6 +103,6 @@ public class ContentHandler { Slime slime = new Slime(); Cursor root = slime.setObject(); root.setString("prepared", request.getUrlBase("/prepared")); - return new SessionResponse(slime, root); + return new SlimeJsonResponse(slime); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java index f761f5f6b6e..57095772b26 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java @@ -2,12 +2,9 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.util.List; /** @@ -16,19 +13,13 @@ import java.util.List; * @author Ulf Lilleengen * @since 5.1 */ -class SessionContentListResponse extends SessionResponse { - private final Slime slime = new Slime(); +class SessionContentListResponse extends SlimeJsonResponse { public SessionContentListResponse(String urlBase, List<ApplicationFile> files) { - super(); Cursor array = slime.setArray(); for (ApplicationFile file : files) { array.addString(urlBase + file.getPath() + (file.isDirectory() ? "/" : "")); } } - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java index 075b4bc329b..08cda869111 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java @@ -3,12 +3,10 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import java.util.logging.Level; + +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.util.*; /** @@ -16,14 +14,11 @@ import java.util.*; * * @author hmusum */ -class SessionContentStatusListResponse extends SessionResponse { +class SessionContentStatusListResponse extends SlimeJsonResponse { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger("SessionContentStatusListResponse"); - private final Slime slime = new Slime(); - public SessionContentStatusListResponse(String urlBase, List<ApplicationFile> files) { - super(); Cursor array = slime.setArray(); for (ApplicationFile f : files) { Cursor element = array.addObject(); @@ -34,9 +29,4 @@ class SessionContentStatusListResponse extends SessionResponse { } } - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java index bd182093e99..e6909e32985 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java @@ -1,28 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; -import java.io.*; /** * Represents a response for a request to show the status and md5sum of a file in the application package. * * @author hmusum */ -public class SessionContentStatusResponse extends SessionResponse { - - private final ApplicationFile file; - private final String urlBase; - private final ApplicationFile.MetaData metaData; - private final ObjectMapper mapper = new ObjectMapper(); +public class SessionContentStatusResponse extends SlimeJsonResponse { public SessionContentStatusResponse(ApplicationFile file, String urlBase) { - super(); - this.file = file; - this.urlBase = urlBase; - ApplicationFile.MetaData metaData; if (file == null) { metaData = new ApplicationFile.MetaData(ApplicationFile.ContentStatusDeleted, ""); @@ -32,24 +23,11 @@ public class SessionContentStatusResponse extends SessionResponse { if (metaData == null) { throw new IllegalArgumentException("Could not find status for '" + file.getPath() + "'"); } - this.metaData = metaData; - } - @Override - public void render(OutputStream outputStream) throws IOException { - mapper.writeValue(outputStream, new ResponseData(metaData.status, metaData.md5, urlBase + file.getPath())); - } - - private static class ResponseData { - public final String status; - public final String md5; - public final String name; - - private ResponseData(String status, String md5, String name) { - this.status = status; - this.md5 = md5; - this.name = name; - } + Cursor element = slime.setObject(); + element.setString("status", metaData.getStatus()); + element.setString("md5", metaData.getMd5()); + element.setString("name", urlBase + file.getPath()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java index 59d12e037e9..fcac023eec3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.TimeoutBudget; import java.time.Clock; @@ -67,14 +64,6 @@ public class SessionHandler extends HttpHandler { return new TimeoutBudget(Clock.systemUTC(), getRequestTimeout(request, defaultTimeout)); } - public static DeployHandlerLogger createLogger(Slime deployLog, HttpRequest request, ApplicationId app) { - return createLogger(deployLog, request.getBooleanProperty("verbose"), app); - } - - public static DeployHandlerLogger createLogger(Slime deployLog, boolean verbose, ApplicationId app) { - return new DeployHandlerLogger(deployLog.get().setArray("log"), verbose, app); - } - /** * True if this request should ignore activation failure because the session was made from an active session that is not active now * @param request a {@link com.yahoo.container.jdisc.HttpRequest} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java deleted file mode 100644 index ad658e3848a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; - -import java.io.IOException; -import java.io.OutputStream; - -import static com.yahoo.jdisc.http.HttpResponse.Status.OK; - -/** - * Superclass for responses from session HTTP handlers. Implements the - * render method. - * - * @author hmusum - * @since 5.1.14 - */ -public class SessionResponse extends HttpResponse { - private final Slime slime; - protected final Cursor root; - - public SessionResponse() { - super(OK); - slime = new Slime(); - root = slime.setObject(); - } - - public SessionResponse(Slime slime, Cursor root) { - super(OK); - this.slime = slime; - this.root = root; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index d6badb8a9a2..9ea96b97af3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -13,11 +13,9 @@ import com.yahoo.vespa.config.server.application.CompressedApplicationInputStrea import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.time.Duration; -import java.time.Instant; import static com.yahoo.vespa.config.server.application.CompressedApplicationInputStream.createFromCompressedStream; import static com.yahoo.vespa.config.server.http.Utils.checkThatTenantExists; @@ -56,7 +54,7 @@ public class ApplicationApiHandler extends SessionHandler { TenantName tenantName = validateTenant(request); PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); CompressedApplicationInputStream compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); - PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams, Instant.now()); + PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams); return new SessionPrepareAndActivateResponse(result, request, prepareParams.getApplicationId(), zone); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java index 95a71881b47..3789939429c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java @@ -3,26 +3,19 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Tenant list response * * @author vegardh */ -public class ListTenantsResponse extends SessionResponse { +public class ListTenantsResponse extends SlimeJsonResponse { ListTenantsResponse(ImmutableSet<TenantName> tenants) { - super(); - Cursor tenantArray = this.root.setArray("tenants"); + Cursor tenantArray = slime.setObject().setArray("tenants"); tenants.forEach(tenantName -> tenantArray.addString(tenantName.value())); } - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java index bb2b57ba45c..24bdfd81f1c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java @@ -1,8 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; /** * Encapsulates the result from preparing an application @@ -13,12 +13,12 @@ public class PrepareResult { private final long sessionId; private final ConfigChangeActions configChangeActions; - private final Slime deployLog; + private final DeployHandlerLogger logger; - public PrepareResult(long sessionId, ConfigChangeActions configChangeActions, Slime deployLog) { + public PrepareResult(long sessionId, ConfigChangeActions configChangeActions, DeployHandlerLogger logger) { this.sessionId = sessionId; this.configChangeActions = configChangeActions; - this.deployLog = deployLog; + this.logger = logger; } public long sessionId() { @@ -29,8 +29,8 @@ public class PrepareResult { return configChangeActions; } - public Slime deployLog() { - return deployLog; + public DeployHandlerLogger deployLogger() { + return logger; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java index 334dbe88614..9c0fbdf2613 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java @@ -4,16 +4,19 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.http.SessionResponse; -public class SessionActiveResponse extends SessionResponse { +public class SessionActiveResponse extends SlimeJsonResponse { public SessionActiveResponse(Slime metaData, HttpRequest request, ApplicationId applicationId, long sessionId, Zone zone) { - super(metaData, metaData.get()); + super(metaData); TenantName tenantName = applicationId.tenant(); String message = "Session " + sessionId + " for tenant '" + tenantName.value() + "' activated."; + Cursor root = metaData.get(); + root.setString("tenant", tenantName.value()); root.setString("message", message); root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 5aee711b379..b0468f5e608 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -11,7 +10,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.application.UriPattern; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.TimeoutBudget; @@ -45,22 +43,25 @@ public class SessionCreateHandler extends SessionHandler { @Override protected HttpResponse handlePOST(HttpRequest request) { - Slime deployLog = applicationRepository.createDeployLog(); final TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); Utils.checkThatTenantExists(applicationRepository.tenantRepository(), tenantName); TimeoutBudget timeoutBudget = SessionHandler.getTimeoutBudget(request, zookeeperBarrierTimeout); - DeployLogger logger = createLogger(request, deployLog, tenantName); + boolean verbose = request.getBooleanProperty("verbose"); + + DeployHandlerLogger logger; long sessionId; if (request.hasProperty("from")) { ApplicationId applicationId = getFromApplicationId(request); + logger = DeployHandlerLogger.forApplication(applicationId, verbose); sessionId = applicationRepository.createSessionFromExisting(applicationId, logger, false, timeoutBudget); } else { validateDataAndHeader(request); + logger = DeployHandlerLogger.forTenant(tenantName, verbose); // TODO: Avoid using application id here at all ApplicationId applicationId = ApplicationId.from(tenantName, ApplicationName.defaultName(), InstanceName.defaultName()); sessionId = applicationRepository.createSession(applicationId, timeoutBudget, request.getData(), request.getHeader(ApplicationApiHandler.contentTypeHeader)); } - return createResponse(request, tenantName, deployLog, sessionId); + return new SessionCreateResponse(logger.slime(), tenantName, request.getHost(), request.getPort(), sessionId); } static ApplicationId getFromApplicationId(HttpRequest request) { @@ -82,11 +83,6 @@ public class SessionCreateHandler extends SessionHandler { .instanceName(match.group(6)).build(); } - private static DeployHandlerLogger createLogger(HttpRequest request, Slime deployLog, TenantName tenant) { - return SessionHandler.createLogger(deployLog, request, - new ApplicationId.Builder().tenant(tenant).applicationName("-").build()); - } - static void validateDataAndHeader(HttpRequest request) { if (request.getData() == null) { throw new BadRequestException("Request contains no data"); @@ -99,9 +95,4 @@ public class SessionCreateHandler extends SessionHandler { ApplicationApiHandler.APPLICATION_X_GZIP + "' and '" + ApplicationApiHandler.APPLICATION_ZIP + "' are supported"); } } - - private HttpResponse createResponse(HttpRequest request, TenantName tenantName, Slime deployLog, long sessionId) { - return new SessionCreateResponse(tenantName, deployLog, deployLog.get()) - .createResponse(request.getHost(), request.getPort(), sessionId); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java index 7d08ea94ce6..33c8f54b1f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java @@ -2,10 +2,9 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionCreateHandler. @@ -13,22 +12,17 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author hmusum * @since 5.1.27 */ -public class SessionCreateResponse extends SessionResponse { - private final TenantName tenantName; +public class SessionCreateResponse extends SlimeJsonResponse { - public SessionCreateResponse(TenantName tenantName, Slime deployLog, Cursor root) { - super(deployLog, root); - this.tenantName = tenantName; - } - - public HttpResponse createResponse(String hostName, int port, long sessionId) { + public SessionCreateResponse(Slime deployLog, TenantName tenantName, String hostName, int port, long sessionId) { + super(deployLog); String path = "http://" + hostName + ":" + port + "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId; + Cursor root = deployLog.get(); - this.root.setString("tenant", tenantName.value()); - this.root.setString("session-id", Long.toString(sessionId)); - this.root.setString("prepared", path + "/prepared"); - this.root.setString("content", path + "/content/"); - this.root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' created."); - return this; + root.setString("tenant", tenantName.value()); + root.setString("session-id", Long.toString(sessionId)); + root.setString("prepared", path + "/prepared"); + root.setString("content", path + "/content/"); + root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' created."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java index 7d9a0b11c28..7bace4749a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java @@ -5,21 +5,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareAndActivateResponse extends SessionResponse { +class SessionPrepareAndActivateResponse extends SlimeJsonResponse { SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { - super(result.deployLog(), result.deployLog().get()); + super(result.deployLogger().slime()); + TenantName tenantName = applicationId.tenant(); String message = "Session " + result.sessionId() + " for tenant '" + tenantName.value() + "' prepared and activated."; - this.root.setString("tenant", tenantName.value()); + Cursor root = slime.get(); + + root.setString("tenant", tenantName.value()); root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenantName + "/application/" + applicationId.application().value() + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index c0789a9c828..258af35be6f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import java.time.Duration; -import java.time.Instant; /** * A handler that prepares a session given by an id in the request. v2 of application API @@ -41,7 +40,7 @@ public class SessionPrepareHandler extends SessionHandler { TenantName tenantName = tenant.getName(); long sessionId = getSessionIdV2(request); PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - PrepareResult result = applicationRepository.prepare(tenant, sessionId, prepareParams, Instant.now()); + PrepareResult result = applicationRepository.prepare(tenant, sessionId, prepareParams); return new SessionPrepareResponse(result, tenantName, request); } @@ -51,7 +50,7 @@ public class SessionPrepareHandler extends SessionHandler { long sessionId = getSessionIdV2(request); applicationRepository.validateThatSessionIsNotActive(tenant, sessionId); applicationRepository.validateThatSessionIsPrepared(tenant, sessionId); - return new SessionPrepareResponse(applicationRepository.createDeployLog(), tenant.getName(), request, sessionId); + return new SessionPrepareResponse(tenant.getName(), request, sessionId); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java index 6d2aa426036..a97cd37d3b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java @@ -3,33 +3,36 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareResponse extends SessionResponse { +class SessionPrepareResponse extends SlimeJsonResponse { - SessionPrepareResponse(Slime deployLog, TenantName tenantName, HttpRequest request, long sessionId) { - this(deployLog, tenantName, request, sessionId, new ConfigChangeActions()); + SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) { + this(new Slime(), tenantName, request, sessionId, new ConfigChangeActions()); } SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) { - this(result.deployLog(), tenantName, request, result.sessionId(), result.configChangeActions()); + this(result.deployLogger().slime(), tenantName, request, result.sessionId(), result.configChangeActions()); } private SessionPrepareResponse(Slime deployLog, TenantName tenantName, HttpRequest request, long sessionId, ConfigChangeActions actions) { - super(deployLog, deployLog.get()); - String message = "Session " + sessionId + " for tenant '" + tenantName.value() + "' prepared."; - this.root.setString("tenant", tenantName.value()); - this.root.setString("activate", "http://" + request.getHost() + ":" + request.getPort() + + super(deployLog); + + Cursor root = deployLog.get().type() != Type.NIX ? deployLog.get() : deployLog.setObject(); + root.setString("tenant", tenantName.value()); + root.setString("activate", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId + "/active"); - root.setString("message", message); + root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' prepared."); new ConfigChangeActionsSlimeConverter(actions).toSlime(root); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java index 2850698ea87..6ff2b30075d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant create @@ -11,16 +10,10 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author vegardh * */ -public class TenantCreateResponse extends SessionResponse { +public class TenantCreateResponse extends MessageResponse { public TenantCreateResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant "+tenant+" created."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant " + tenant + " created."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java index 3ba61f84270..d21584c8cdc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant delete @@ -10,16 +9,10 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author vegardh * */ -public class TenantDeleteResponse extends SessionResponse { +public class TenantDeleteResponse extends MessageResponse { public TenantDeleteResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant "+tenant+" deleted."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant " + tenant + " deleted."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java index 183fcd9b3c1..b918cab7828 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java @@ -2,24 +2,17 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant create * * @author hmusum */ -public class TenantGetResponse extends SessionResponse { +public class TenantGetResponse extends MessageResponse { public TenantGetResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant '" + tenant + "' exists."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant '" + tenant + "' exists."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index 7ab55b9af72..9820eac2f30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -52,7 +52,9 @@ public class DelayedConfigResponses { // Since JRT does not allow adding watcher for "fake" requests, we must be able to disable it for unit tests :( DelayedConfigResponses(RpcServer rpcServer, int numTimerThreads, boolean useJrtWatcher) { this.rpcServer = rpcServer; - ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(numTimerThreads, ThreadFactoryFactory.getThreadFactory(DelayedConfigResponses.class.getName())); + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor(numTimerThreads, + ThreadFactoryFactory.getDaemonThreadFactory("delayed config responses")); executor.setRemoveOnCancelPolicy(true); this.executorService = executor; this.useJrtWatcher = useJrtWatcher; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index a96ae16a20b..55ee45482f9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -133,7 +133,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); int rpcWorkerThreads = (config.numRpcThreads() == 0) ? threadsToUse() : config.numRpcThreads(); executorService = new ThreadPoolExecutor(rpcWorkerThreads, rpcWorkerThreads, - 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); + 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getDaemonThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); spec = new Spec(null, config.rpcport()); hostRegistry = hostRegistries.getTenantHostRegistry(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index d962218b63a..914c927698f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -45,6 +45,7 @@ public final class PrepareParams { static final String APPLICATION_CONTAINER_ROLE = "applicationContainerRole"; static final String QUOTA_PARAM_NAME = "quota"; static final String FORCE_PARAM_NAME = "force"; + static final String INTERNAL_RESTART_PARAM_NAME = "internalRestart"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -53,6 +54,7 @@ public final class PrepareParams { private final boolean verbose; private final boolean isBootstrap; private final boolean force; + private final boolean internalRestart; private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; @@ -67,7 +69,8 @@ public final class PrepareParams { List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, - Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) { + Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force, + boolean internalRestart) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); this.ignoreValidationErrors = ignoreValidationErrors; @@ -83,6 +86,7 @@ public final class PrepareParams { this.applicationRoles = applicationRoles; this.quota = quota; this.force = force; + this.internalRestart = internalRestart; } public static class Builder { @@ -92,6 +96,7 @@ public final class PrepareParams { private boolean verbose = false; private boolean isBootstrap = false; private boolean force = false; + private boolean internalRestart = false; private ApplicationId applicationId = null; private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); @@ -208,11 +213,16 @@ public final class PrepareParams { return this; } + public Builder internalRestart(boolean internalRestart) { + this.internalRestart = internalRestart; + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata, dockerImageRepository, athenzDomain, - applicationRoles, quota, force); + applicationRoles, quota, force, internalRestart); } } @@ -231,6 +241,7 @@ public final class PrepareParams { .applicationRoles(ApplicationRoles.fromString(request.getProperty(APPLICATION_HOST_ROLE), request.getProperty(APPLICATION_CONTAINER_ROLE))) .quota(request.getProperty(QUOTA_PARAM_NAME)) .force(request.getBooleanProperty(FORCE_PARAM_NAME)) + .internalRestart(request.getBooleanProperty(INTERNAL_RESTART_PARAM_NAME)) .build(); } @@ -282,6 +293,8 @@ public final class PrepareParams { public boolean force() { return force; } + public boolean internalRestart() { return internalRestart; } + public TimeoutBudget getTimeoutBudget() { return timeoutBudget; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 4f577d8f62c..78bbe8b5438 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -47,7 +47,6 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -123,10 +122,11 @@ public class SessionRepository { // ---------------- Local sessions ---------------------------------------------------------------- public synchronized void addLocalSession(LocalSession session) { - localSessionCache.put(session.getSessionId(), session); long sessionId = session.getSessionId(); - RemoteSession remoteSession = createRemoteSession(sessionId); - addSessionStateWatcher(sessionId, remoteSession); + localSessionCache.put(sessionId, session); + if (remoteSessionCache.get(sessionId) == null) { + createRemoteSession(sessionId); + } } public LocalSession getLocalSession(long sessionId) { @@ -265,11 +265,6 @@ public class SessionRepository { return getSessionList(curator.getChildren(sessionsPath)); } - public void addRemoteSession(RemoteSession session) { - remoteSessionCache.put(session.getSessionId(), session); - metrics.incAddedSessions(); - } - public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { int deleted = 0; for (long sessionId : getRemoteSessions()) { @@ -343,17 +338,20 @@ public class SessionRepository { * * @param sessionId session id for the new session */ - public void sessionAdded(long sessionId) { + public synchronized void sessionAdded(long sessionId) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); if (sessionZKClient.readStatus().equals(Session.Status.DELETE)) return; - log.log(Level.FINE, () -> "Adding remote session to SessionRepository: " + sessionId); - RemoteSession remoteSession = createRemoteSession(sessionId); - loadSessionIfActive(remoteSession); - addRemoteSession(remoteSession); + log.log(Level.FINE, () -> "Adding remote session " + sessionId); + RemoteSession session = createRemoteSession(sessionId); + if (session.getStatus() == Session.Status.NEW) { + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); + session.confirmUpload(); + } else { + log.log(Level.WARNING, () -> session.logPre() + "Session " + sessionId + " added, but with unexpected status " + session.getStatus()); + } if (distributeApplicationPackage()) createLocalSessionUsingDistributedApplicationPackage(sessionId); - addSessionStateWatcher(sessionId, remoteSession); } void activate(RemoteSession session) { @@ -373,14 +371,20 @@ public class SessionRepository { } public void delete(RemoteSession remoteSession) { - LocalSession localSession = getLocalSession(remoteSession.getSessionId()); - remoteSession.deactivate(); - if (localSession == null) { - // This change will be picked up by directoryCache in this class, which will do the rest of the cleanup - try (Lock lock = lock(remoteSession.getSessionId())) { - remoteSession.delete(); - } - } else { + // This change will be picked up by directoryCache in this class, which will do the rest of + // the cleanup on all config servers + long sessionId = remoteSession.getSessionId(); + try (Lock lock = lock(sessionId)) { + // TODO: Change log level to FINE when debugging is finished + log.log(Level.INFO, () -> remoteSession.logPre() + "Deactivating and deleting remote session " + sessionId); + remoteSession.deactivate(); + remoteSession.delete(); + remoteSessionCache.remove(sessionId); + } + LocalSession localSession = getLocalSession(sessionId); + if (localSession != null) { + // TODO: Change log level to FINE when debugging is finished + log.log(Level.INFO, () -> localSession.logPre() + "Deleting local session " + sessionId); deleteLocalSession(localSession); } } @@ -430,26 +434,16 @@ public class SessionRepository { log.log(Level.FINE, () -> "Got child event: " + event); switch (event.getType()) { case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); - break; case CHILD_REMOVED: case CONNECTION_RECONNECTED: sessionsChanged(); break; + default: + break; } }); } - private void synchronizeOnNew(List<Long> sessionList) { - for (long sessionId : sessionList) { - RemoteSession session = remoteSessionCache.get(sessionId); - if (session == null) continue; // session might have been deleted after getting session list - log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); - session.confirmUpload(); - } - } - /** * Creates a new deployment session from an application package. * @@ -463,9 +457,13 @@ public class SessionRepository { return create(applicationDirectory, applicationId, activeSessionId, false, timeoutBudget); } - public RemoteSession createRemoteSession(long sessionId) { + public synchronized RemoteSession createRemoteSession(long sessionId) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); - return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + RemoteSession session = new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + remoteSessionCache.put(sessionId, session); + loadSessionIfActive(session); + addSessionStateWatcher(sessionId, session); + return session; } private void ensureSessionPathDoesNotExist(long sessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index c6c08beea17..d6d08aaac6c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -24,18 +24,18 @@ public class SessionStateWatcher { private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); private final Curator.FileCache fileCache; - private final RemoteSession remoteSession; + private final RemoteSession session; private final MetricUpdater metrics; private final Executor zkWatcherExecutor; private final SessionRepository sessionRepository; SessionStateWatcher(Curator.FileCache fileCache, - RemoteSession remoteSession, + RemoteSession session, MetricUpdater metrics, Executor zkWatcherExecutor, SessionRepository sessionRepository) { this.fileCache = fileCache; - this.remoteSession = remoteSession; + this.session = session; this.metrics = metrics; this.fileCache.addListener(this::nodeChanged); this.fileCache.start(); @@ -44,24 +44,24 @@ public class SessionStateWatcher { } private void sessionStatusChanged(Status newStatus) { - long sessionId = remoteSession.getSessionId(); + long sessionId = session.getSessionId(); switch (newStatus) { case NEW: case NONE: break; case PREPARE: createLocalSession(sessionId); - sessionRepository.prepare(remoteSession); + sessionRepository.prepare(session); break; case ACTIVATE: createLocalSession(sessionId); - sessionRepository.activate(remoteSession); + sessionRepository.activate(session); break; case DEACTIVATE: - sessionRepository.deactivate(remoteSession); + sessionRepository.deactivate(session); break; case DELETE: - sessionRepository.delete(remoteSession); + sessionRepository.delete(session); break; default: throw new IllegalStateException("Unknown status " + newStatus); @@ -75,7 +75,7 @@ public class SessionStateWatcher { } public long getSessionId() { - return remoteSession.getSessionId(); + return session.getSessionId(); } public void close() { @@ -93,12 +93,12 @@ public class SessionStateWatcher { ChildData node = fileCache.getCurrentData(); if (node != null) { newStatus = Status.parse(Utf8.toString(node.getData())); - log.log(Level.FINE, remoteSession.logPre() + "Session change: Session " - + remoteSession.getSessionId() + " changed status to " + newStatus.name()); + log.log(Level.FINE, session.logPre() + "Session change: Session " + + session.getSessionId() + " changed status to " + newStatus.name()); sessionStatusChanged(newStatus); } } catch (Exception e) { - log.log(Level.WARNING, remoteSession.logPre() + "Error handling session change to " + + log.log(Level.WARNING, session.logPre() + "Error handling session change to " + newStatus.name() + " for session " + getSessionId(), e); metrics.incSessionChangeErrors(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 41377bdf317..57e49fe365a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -84,7 +85,8 @@ public class TenantRepository { private final ExecutorService zkCacheExecutor; private final StripedExecutor<TenantName> zkWatcherExecutor; private final ExecutorService bootstrapExecutor; - private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); + private final ScheduledExecutorService checkForRemovedApplicationsService = + new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications")); private final Optional<Curator.DirectoryCache> directoryCache; /** @@ -96,7 +98,8 @@ public class TenantRepository { public TenantRepository(GlobalComponentRegistry componentRegistry) { this.componentRegistry = componentRegistry; ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); - this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); + this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), + new DaemonThreadFactory("bootstrap tenants")); this.curator = componentRegistry.getCurator(); metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(componentRegistry.getTenantListener()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java index 641aa31a6b7..b0bc8dc90dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java @@ -5,29 +5,44 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.Optional; /** - * Contains version information for this configserver. + * + * Contains version information for this configserver. Stored both in file system and in ZooKeeper (uses + * data in ZooKeeper if distributeApplicationPackage and data found in ZooKeeper) * * @author Ulf Lilleengen */ public class VersionState { + static final Path versionPath = Path.fromString("/config/v2/vespa_version"); + private final File versionFile; + private final Curator curator; + private final BooleanFlag distributeApplicationPackage; @Inject - public VersionState(ConfigserverConfig config) { - this(new File(Defaults.getDefaults().underVespaHome(config.configServerDBDir()), "vespa_version")); + public VersionState(ConfigserverConfig config, Curator curator, FlagSource flagsource) { + this(new File(Defaults.getDefaults().underVespaHome(config.configServerDBDir()), "vespa_version"), curator, flagsource); } - public VersionState(File versionFile) { + public VersionState(File versionFile, Curator curator, FlagSource flagSource) { this.versionFile = versionFile; + this.curator = curator; + this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); } public boolean isUpgraded() { @@ -35,14 +50,29 @@ public class VersionState { } public void saveNewVersion() { + saveNewVersion(currentVersion().toFullString()); + } + + public void saveNewVersion(String vespaVersion) { + curator.set(versionPath, Utf8.toBytes(vespaVersion)); try (FileWriter writer = new FileWriter(versionFile)) { - writer.write(currentVersion().toFullString()); + writer.write(vespaVersion); } catch (IOException e) { throw new RuntimeException(e); } } public Version storedVersion() { + if (distributeApplicationPackage.value()) { + Optional<byte[]> version = curator.getData(versionPath); + if(version.isPresent()) { + try { + return Version.fromString(Utf8.toString(version.get())); + } catch (Exception e) { + // continue, use value in file + } + } + } try (FileReader reader = new FileReader(versionFile)) { return Version.fromString(IOUtils.readAll(reader)); } catch (Exception e) { @@ -54,6 +84,10 @@ public class VersionState { return new Version(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); } + File versionFile() { + return versionFile; + } + @Override public String toString() { return String.format("Current version:%s, stored version:%s", currentVersion(), storedVersion()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 4770b0797eb..e90ef38a92f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -382,8 +382,9 @@ public class ApplicationRepositoryTest { new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(serverdb.getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .sessionLifetime(60)); - DeployTester tester = new DeployTester(configserverConfig, clock); + DeployTester tester = new DeployTester.Builder().configserverConfig(configserverConfig).clock(clock).build(); tester.deployApp("src/test/apps/app", clock.instant()); // session 2 (numbering starts at 2) clock.advance(Duration.ofSeconds(10)); @@ -558,7 +559,7 @@ public class ApplicationRepositoryTest { long firstSession = result.sessionId(); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams()); exceptionRule.expect(RuntimeException.class); exceptionRule.expectMessage(containsString("Timeout exceeded when trying to activate 'test1.testapp'")); applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0)), false); @@ -583,7 +584,7 @@ public class ApplicationRepositoryTest { PrepareResult result2 = deployApp(testAppJdiscOnly); result2.sessionId(); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId2, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId2, prepareParams()); exceptionRule.expect(ActivationConflictException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Cannot activate session 3 because the currently active session (4) has changed since session 3 was created (was 2 at creation time)")); applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId2, timeoutBudget, false); @@ -596,7 +597,7 @@ public class ApplicationRepositoryTest { exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("Session is active: 2")); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams()); exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Session 2 is already active")); @@ -704,7 +705,7 @@ public class ApplicationRepositoryTest { } private PrepareResult prepareAndActivate(File application) { - return applicationRepository.deploy(application, prepareParams(), Instant.now()); + return applicationRepository.deploy(application, prepareParams()); } private PrepareResult deployApp(File applicationPackage) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 23323d11f76..73f38188ec9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -25,6 +24,7 @@ import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -32,7 +32,6 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -56,6 +55,8 @@ import static org.junit.Assert.assertTrue; */ public class ConfigServerBootstrapTest { + private final MockCurator curator = new MockCurator(); + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -63,11 +64,11 @@ public class ConfigServerBootstrapTest { public void testBootstrap() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); @@ -96,11 +97,11 @@ public class ConfigServerBootstrapTest { public void testBootstrapWithVipStatusFile() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); @@ -121,11 +122,11 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapWhenRedeploymentFails() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).build(); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); // Manipulate application package so that it will fail deployment when config server starts @@ -158,18 +159,19 @@ public class ConfigServerBootstrapTest { public void testBootstrapNonHostedOneConfigModel() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfigNonHosted(temporaryFolder); String vespaVersion = "1.2.3"; - List<ModelFactory> modelFactories = Collections.singletonList(DeployTester.createModelFactory(Version.fromString(vespaVersion))); List<Host> hosts = createHosts(vespaVersion); - InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); Curator curator = new MockCurator(); - DeployTester tester = new DeployTester(modelFactories, configserverConfig, - Clock.systemUTC(), new Zone(Environment.dev, RegionName.defaultName()), - provisioner, curator); + DeployTester tester = new DeployTester.Builder() + .modelFactory(DeployTester.createModelFactory(Version.fromString(vespaVersion))) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true)) + .configserverConfig(configserverConfig) + .zone(new Zone(Environment.dev, RegionName.defaultName())) + .curator(curator) + .build(); tester.deployApp("src/test/apps/app/", vespaVersion, Instant.now()); ApplicationId applicationId = tester.applicationId(); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); // Ugly hack, but I see no other way of doing it: @@ -220,6 +222,7 @@ public class ConfigServerBootstrapTest { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .hostedVespa(hosted) .multitenant(hosted) .maxDurationOfBootstrap(2) /* seconds */ @@ -241,6 +244,10 @@ public class ConfigServerBootstrapTest { stateMonitor); } + private VersionState createVersionState() throws IOException { + return new VersionState(temporaryFolder.newFile(), curator, new InMemoryFlagSource()); + } + public static class MockRpcServer extends com.yahoo.vespa.config.server.rpc.MockRpcServer { volatile boolean isRunning = false; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index e2c3369d49e..ead1e79a416 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.configchange; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import java.util.ArrayList; import java.util.List; @@ -22,11 +23,17 @@ public class ConfigChangeActionsBuilder { } public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName) { - actions.add(new MockRestartAction(message, - List.of(createService(clusterName, clusterType, serviceType, serviceName)))); + return restart(message, clusterName, clusterType, serviceType, serviceName, false); + } + + public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName, boolean ignoreForInternalRedeploy) { + actions.add(new VespaRestartAction(message, + createService(clusterName, clusterType, serviceType, serviceName), + ignoreForInternalRedeploy)); return this; } + ConfigChangeActionsBuilder refeed(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { actions.add(new MockRefeedAction(name, allowed, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java index bdf63befd15..904ca10aa1c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java @@ -30,6 +30,11 @@ public class MockRefeedAction extends MockConfigChangeAction implements ConfigCh public boolean allowed() { return allowed; } @Override + public boolean ignoreForInternalRedeploy() { + return false; + } + + @Override public String getDocumentType() { return documentType; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java deleted file mode 100644 index b1183f91282..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.configchange; - -import com.yahoo.config.model.api.ConfigChangeRestartAction; -import com.yahoo.config.model.api.ServiceInfo; - -import java.util.List; - -/** - * @author geirst - * @since 5.44 - */ -public class MockRestartAction extends MockConfigChangeAction implements ConfigChangeRestartAction { - public MockRestartAction(String message, List<ServiceInfo> services) { - super(message, services); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java index ee0180802af..c19b81aa91b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java @@ -5,8 +5,10 @@ import com.yahoo.config.model.api.ServiceInfo; import org.junit.Test; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -81,4 +83,18 @@ public class RestartActionsTest { assertThat(toString(entries.get(0)), equalTo("content.foo.searchnode:[baz][change]")); assertThat(toString(entries.get(1)), equalTo("search.foo.searchnode:[baz][change]")); } + + @Test + public void use_for_internal_restart_test() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder() + .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME) + .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE_2, SERVICE_TYPE, SERVICE_NAME, true).build(); + + assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), + actions.getRestartActions().getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), + actions.getRestartActions().useForInternalRestart(false).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + assertEquals(Set.of(CLUSTER_TYPE), + actions.getRestartActions().useForInternalRestart(true).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index 6aa72e3e672..3ac9e681604 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -6,9 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import java.util.logging.Level; import com.yahoo.log.LogLevel; -import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; import org.junit.Test; @@ -33,13 +31,11 @@ public class DeployHandlerLoggerTest { } private void testLogging(boolean verbose, String expectedPattern) throws IOException { - Slime slime = new Slime(); - Cursor array = slime.setArray(); - DeployLogger logger = new DeployHandlerLogger(array, verbose, new ApplicationId.Builder() - .tenant("testtenant").applicationName("testapp").build()); + DeployHandlerLogger logger = DeployHandlerLogger.forApplication( + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build(), verbose); logMessages(logger); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new JsonFormat(true).encode(baos, slime); + new JsonFormat(true).encode(baos, logger.slime()); assertTrue(Pattern.matches(expectedPattern, baos.toString())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index d92245bf5c1..7553583e70c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -42,6 +42,7 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; +import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; import java.nio.file.Files; @@ -70,72 +71,16 @@ public class DeployTester { private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - public DeployTester() { - this(Collections.singletonList(createModelFactory(Clock.systemUTC()))); - } - - public DeployTester(List<ModelFactory> modelFactories) { - this(modelFactories, - new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) - .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString())), - Clock.systemUTC()); - } - - public DeployTester(ConfigserverConfig configserverConfig, Clock clock) { - this(Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { - this(modelFactories, configserverConfig, Clock.systemUTC()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { - this(modelFactories, configserverConfig, clock, Zone.defaultZone()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, HostProvisioner hostProvisioner) { - this(modelFactories, configserverConfig, Clock.systemUTC(), hostProvisioner); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) { - this(modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) { - this(modelFactories, configserverConfig, clock, zone, createProvisioner()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { - this(modelFactories, configserverConfig, clock, zone, provisioner, new MockCurator()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, - HostProvisioner provisioner, Curator curator) { + private DeployTester(Clock clock, TenantRepository tenantRepository, ApplicationRepository applicationRepository) { this.clock = clock; - TestComponentRegistry componentRegistry = createComponentRegistry(curator, Metrics.createTestMetrics(), - modelFactories, configserverConfig, clock, zone, - provisioner); - try { - this.tenantRepository = new TenantRepository(componentRegistry); - tenantRepository.addTenant(tenantName); - } - catch (Exception e) { - throw new IllegalArgumentException(e); - } - applicationRepository = new ApplicationRepository.Builder() - .withTenantRepository(tenantRepository) - .withProvisioner(new ProvisionerAdapter(provisioner)) - .withConfigserverConfig(configserverConfig) - .withOrchestrator(new OrchestratorMock()) - .withClock(clock) - .build(); + this.tenantRepository = tenantRepository; + this.applicationRepository = applicationRepository; } public Tenant tenant() { return tenantRepository.getTenant(tenantName); } - + /** Create a model factory for the version of this source*/ public static CountingModelFactory createModelFactory(Clock clock) { return new CountingModelFactory(clock); @@ -215,7 +160,7 @@ public class DeployTester { paramsBuilder.applicationId(applicationId) .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); - return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), now); + return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build()); } public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { @@ -296,11 +241,11 @@ public class DeployTester { private static class FailingModelFactory implements ModelFactory { private final Version version; - + public FailingModelFactory(Version version) { this.version = version; } - + @Override public Version version() { return version; } @@ -369,4 +314,99 @@ public class DeployTester { } + public static class Builder { + private Clock clock; + private Provisioner provisioner; + private ConfigserverConfig configserverConfig; + private Zone zone; + private Curator curator; + private Metrics metrics; + private List<ModelFactory> modelFactories; + private Orchestrator orchestrator; + + public DeployTester build() { + Clock clock = Optional.ofNullable(this.clock).orElseGet(Clock::systemUTC); + Zone zone = Optional.ofNullable(this.zone).orElseGet(Zone::defaultZone); + ConfigserverConfig configserverConfig = Optional.ofNullable(this.configserverConfig) + .orElseGet(() -> new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) + .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) + .fileReferencesDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()))); + Provisioner provisioner = Optional.ofNullable(this.provisioner) + .orElseGet(() -> new ProvisionerAdapter(createProvisioner())); + List<ModelFactory> modelFactories = Optional.ofNullable(this.modelFactories) + .orElseGet(() -> List.of(createModelFactory(clock))); + + TestComponentRegistry.Builder testComponentRegistryBuilder = new TestComponentRegistry.Builder() + .clock(clock) + .configServerConfig(configserverConfig) + .curator(Optional.ofNullable(curator).orElseGet(MockCurator::new)) + .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) + .metrics(Optional.ofNullable(metrics).orElseGet(Metrics::createTestMetrics)) + .zone(zone); + if (configserverConfig.hostedVespa()) testComponentRegistryBuilder.provisioner(provisioner); + + TenantRepository tenantRepository = new TenantRepository(testComponentRegistryBuilder.build()); + tenantRepository.addTenant(tenantName); + + ApplicationRepository applicationRepository = new ApplicationRepository.Builder() + .withTenantRepository(tenantRepository) + .withConfigserverConfig(configserverConfig) + .withOrchestrator(Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new)) + .withClock(clock) + .withProvisioner(provisioner) + .build(); + + return new DeployTester(clock, tenantRepository, applicationRepository); + } + + public Builder clock(Clock clock) { + this.clock = clock; + return this; + } + + public Builder provisioner(Provisioner provisioner) { + this.provisioner = provisioner; + return this; + } + + public Builder hostProvisioner(HostProvisioner hostProvisioner) { + return provisioner(new ProvisionerAdapter(hostProvisioner)); + } + + public Builder configserverConfig(ConfigserverConfig configserverConfig) { + this.configserverConfig = configserverConfig; + return this; + } + + public Builder zone(Zone zone) { + this.zone = zone; + return this; + } + + public Builder curator(Curator curator) { + this.curator = curator; + return this; + } + + public Builder metrics(Metrics metrics) { + this.metrics = metrics; + return this; + } + + public Builder modelFactory(ModelFactory modelFactory) { + return modelFactories(List.of(modelFactory)); + } + + public Builder modelFactories(List<ModelFactory> modelFactories) { + this.modelFactories = modelFactories; + return this; + } + + public Builder orchestrator(Orchestrator orchestrator) { + this.orchestrator = orchestrator; + return this; + } + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 254fe62cba8..18ca54e8c11 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -20,12 +20,12 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.config.server.configchange.MockRestartAction; import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -65,8 +65,9 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() throws IOException { - CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) + .configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/hosted/", "4.5.6"); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId()); @@ -77,7 +78,9 @@ public class HostedDeployTest { @Test public void testRedeploy() throws IOException { - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory()) + .configserverConfig(createConfigserverConfig()).build(); ApplicationId appId = tester.applicationId(); tester.deployApp("src/test/apps/hosted/"); assertFalse(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); @@ -90,8 +93,9 @@ public class HostedDeployTest { @Test public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() throws IOException { - CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) + .configserverConfig(createConfigserverConfig()).build(); String dockerImageRepository = "docker.foo.com:4443/bar/baz"; tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder() .vespaVersion("4.5.6") @@ -111,7 +115,7 @@ public class HostedDeployTest { List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("6.1.0")), createHostedModelFactory(Version.fromString("6.2.0")), createHostedModelFactory(Version.fromString("7.0.0"))); - DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/hosted/", "6.2.0"); assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); } @@ -332,7 +336,7 @@ public class HostedDeployTest { ManualClock clock = new ManualClock("2016-10-09T00:00:00"); List<ModelFactory> modelFactories = List.of(createHostedModelFactory(clock), createFailingModelFactory(Version.fromString("1.0.0"))); // older than default - DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/validationOverride/", clock.instant()); // Redeployment from local active works @@ -373,8 +377,8 @@ public class HostedDeployTest { new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); List<ModelFactory> modelFactories = List.of( - new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), new MockRestartAction("change", services)), - new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), new MockRestartAction("other change", services))); + new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), new VespaRestartAction("change", services)), + new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), new VespaRestartAction("other change", services))); DeployTester tester = createTester(hosts, modelFactories, prodZone); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.2.0"); @@ -393,6 +397,7 @@ public class HostedDeployTest { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .hostedVespa(true) .multitenant(true) .region(zone.region().value()) @@ -415,8 +420,12 @@ public class HostedDeployTest { private DeployTester createTester(List<Host> hosts, List<ModelFactory> modelFactories, Zone prodZone, Clock clock) throws IOException { - return new DeployTester(modelFactories, createConfigserverConfig(prodZone), - clock, prodZone, new InMemoryProvisioner(new Hosts(hosts), true)); + return new DeployTester.Builder() + .modelFactories(modelFactories) + .configserverConfig(createConfigserverConfig(prodZone)) + .clock(clock) + .zone(prodZone) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true)).build(); } private static class ConfigChangeActionsModelFactory extends TestModelFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index c07c7316930..015cc039a1c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -27,7 +27,7 @@ public class RedeployTest { @Test public void testRedeploy() { - DeployTester tester = new DeployTester(); + DeployTester tester = new DeployTester.Builder().build(); tester.deployApp("src/test/apps/app"); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); @@ -45,7 +45,7 @@ public class RedeployTest { public void testNoRedeploy() { List<ModelFactory> modelFactories = List.of(createModelFactory(Clock.systemUTC()), createFailingModelFactory(Version.fromString("1.0.0"))); - DeployTester tester = new DeployTester(modelFactories); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).build(); ApplicationId id = ApplicationId.from(tester.tenant().getName(), ApplicationName.from("default"), InstanceName.from("default")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 40aee72e71c..ea896469f03 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -48,12 +48,14 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -61,6 +63,7 @@ public class HttpGetConfigHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index 906716125c2..9cf8f7960a3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -53,12 +53,14 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -66,6 +68,7 @@ public class HttpListConfigsHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 67ac0b02133..ab5303b221e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -17,7 +18,9 @@ import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -35,8 +38,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private static final File testApp = new File("src/test/apps/content"); private static final File testApp2 = new File("src/test/apps/content2"); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - private final Clock clock = componentRegistry.getClock(); + private final TenantName tenantName1 = TenantName.from("mofet"); private final TenantName tenantName2 = TenantName.from("bla"); @@ -48,8 +50,22 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private ApplicationRepository applicationRepository; private ApplicationHandler handler; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupHandler() { + public void setupHandler() throws IOException { + + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .configServerConfig(configserverConfig) + .build(); + Clock clock = componentRegistry.getClock(); + TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName1); tenantRepository.addTenant(tenantName2); @@ -59,6 +75,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams(appId1)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index d84b81bd8e7..9e4ca8dfb2c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; @@ -31,7 +32,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import javax.ws.rs.client.Client; import java.io.ByteArrayInputStream; @@ -73,12 +76,21 @@ public class ApplicationHandlerTest { private SessionHandlerTest.MockProvisioner provisioner; private OrchestratorMock orchestrator; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { List<ModelFactory> modelFactories = List.of(DeployTester.createModelFactory(vespaVersion)); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .provisioner(provisioner) .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) + .configServerConfig(configserverConfig) .build(); tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(mytenantName); @@ -91,6 +103,7 @@ public class ApplicationHandlerTest { .withClock(componentRegistry.getClock()) .withTesterClient(testerClient) .withLogRetriever(logRetriever) + .withConfigserverConfig(configserverConfig) .build(); } @@ -357,7 +370,7 @@ public class ApplicationHandlerTest { assertEquals(200, response.getStatus()); String renderedString = SessionHandlerTest.getRenderedString(response); assertEquals("{\"generation\":" + expectedGeneration + - ",\"applicationPackageFileReference\":\"\"" + + ",\"applicationPackageFileReference\":\"./\"" + ",\"modelVersions\":[\"" + expectedVersion.toFullString() + "\"]}", renderedString); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 4d64721f7dd..6b9abf5d7ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -19,7 +20,9 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -40,10 +43,19 @@ public class HostHandlerTest { private final static Zone zone = Zone.defaultZone(); private ApplicationRepository applicationRepository; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .zone(zone) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(mytenant); @@ -51,6 +63,7 @@ public class HostHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HostHandler(HostHandler.testOnlyContext(), applicationRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 14d7a0743a5..9bb113875b4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -57,12 +57,14 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -70,6 +72,7 @@ public class HttpGetConfigHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 785c9977fd2..c1adec3336d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -61,12 +61,14 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -74,6 +76,7 @@ public class HttpListConfigsHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams()); handler = new HttpListConfigsHandler(HttpListConfigsHandler.testOnlyContext(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index c3a7e82dff5..511717acfc0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.provision.ApplicationId; @@ -65,12 +66,18 @@ public class SessionActiveHandlerTest { public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setup() { + public void setup() throws IOException { VespaModelFactory modelFactory = new TestModelFactory(Version.fromString("7.222.2")); hostProvisioner = new SessionHandlerTest.MockProvisioner(); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); componentRegistry = new TestComponentRegistry.Builder() .curator(new MockCurator()) .modelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); @@ -79,6 +86,7 @@ public class SessionActiveHandlerTest { .withProvisioner(hostProvisioner) .withOrchestrator(new OrchestratorMock()) .withClock(componentRegistry.getClock()) + .withConfigserverConfig(configserverConfig) .build(); handler = createHandler(); } @@ -133,8 +141,7 @@ public class SessionActiveHandlerTest { testApp); applicationRepository.prepare(tenant, sessionId, - new PrepareParams.Builder().applicationId(applicationId()).build(), - componentRegistry.getClock().instant()); + new PrepareParams.Builder().applicationId(applicationId()).build()); actResponse = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); LocalSession session = applicationRepository.getActiveLocalSession(tenant, applicationId()); metaData = session.getMetaData(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index d28404d8d72..6de85f12765 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -19,7 +20,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayInputStream; import java.io.File; @@ -37,14 +40,25 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { private static final TenantName tenantName = TenantName.from("contenttest"); private static final File testApp = new File("src/test/apps/content"); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - + private TestComponentRegistry componentRegistry; private TenantRepository tenantRepository; private SessionContentHandler handler = null; private long sessionId; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupHandler() { + public void setupHandler() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + componentRegistry = new TestComponentRegistry.Builder() + .configServerConfig(configserverConfig) + .build(); + tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); @@ -52,6 +66,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId()).build()); Tenant tenant = applicationRepository.getTenant(applicationId()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index cc4f39b0789..f1abddba63c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -25,7 +25,9 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.File; @@ -54,17 +56,30 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { private static final File app = new File("src/test/resources/deploy/validapp"); private final Curator curator = new MockCurator(); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - private final Clock clock = componentRegistry.getClock(); - private final TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); + private TimeoutBudget timeoutBudget; private ApplicationRepository applicationRepository; + private TestComponentRegistry componentRegistry; private String preparedMessage = " prepared.\"}"; private String tenantMessage = ""; private TenantRepository tenantRepository; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupRepo() { + public void setupRepo() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(configserverConfig) + .build(); + Clock clock = componentRegistry.getClock(); + timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); applicationRepository = new ApplicationRepository.Builder() @@ -72,6 +87,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) + .withConfigserverConfig(configserverConfig) .build(); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; preparedMessage = " for tenant '" + tenant + "' prepared.\""; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index ecab121e547..748c43bafeb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -8,29 +8,31 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.Clock; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.SessionResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.NotFoundException; +import org.junit.rules.TemporaryFolder; public class TenantHandlerTest { @@ -41,13 +43,26 @@ public class TenantHandlerTest { private TenantHandler handler; private final TenantName a = TenantName.from("a"); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { - tenantRepository = new TenantRepository(new TestComponentRegistry.Builder().curator(new MockCurator()).build()); + public void setup() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() + .curator(new MockCurator()) + .configServerConfig(configserverConfig) + .build()); + applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new TenantHandler(TenantHandler.testOnlyContext(), applicationRepository); } @@ -141,7 +156,7 @@ public class TenantHandlerTest { return (TenantCreateResponse) handler.handlePUT(testRequest); } - private void assertResponseEquals(SessionResponse response, String payload) throws IOException { + private void assertResponseEquals(HttpResponse response, String payload) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); assertEquals(baos.toString(StandardCharsets.UTF_8), payload); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index 78d69b75d59..712242a69e6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -36,10 +36,8 @@ class MaintainerTester { private final Curator curator; private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - private final Clock clock; MaintainerTester(Clock clock, TemporaryFolder temporaryFolder) throws IOException { - this.clock = clock; this.curator = new MockCurator(); InMemoryProvisioner hostProvisioner = new InMemoryProvisioner(true, "host0", "host1", "host2", "host3", "host4"); ProvisionerAdapter provisioner = new ProvisionerAdapter(hostProvisioner); @@ -47,6 +45,7 @@ class MaintainerTester { .hostedVespa(true) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) @@ -67,7 +66,7 @@ class MaintainerTester { } void deployApp(File applicationPath, PrepareParams.Builder prepareParams) { - applicationRepository.deploy(applicationPath, prepareParams.ignoreValidationErrors(true).build(), clock.instant()); + applicationRepository.deploy(applicationPath, prepareParams.ignoreValidationErrors(true).build()); } Curator curator() { return curator; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index eb06f2f7017..47217491e3c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -82,7 +82,8 @@ public class RpcTester implements AutoCloseable { spec = createSpec(port); configBuilder.rpcport(port) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()); + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()); configserverConfig = new ConfigserverConfig(configBuilder); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java deleted file mode 100644 index fb268492dd7..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.Version; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.time.Instant; -import java.util.Collections; -import java.util.Optional; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class LocalSessionTest { - - private static final File testApp = new File("src/test/apps/app"); - private static final TenantName tenantName = TenantName.from("test_tenant"); - private static final Path tenantPath = Path.createRoot(); - - private TenantRepository tenantRepository; - private Curator curator; - private ConfigCurator configCurator; - - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before - public void setupTest() throws IOException { - curator = new MockCurator(); - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .configServerConfig(new ConfigserverConfig.Builder() - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) - .build(); - tenantRepository = new TenantRepository(componentRegistry); - tenantRepository.addTenant(tenantName); - configCurator = ConfigCurator.create(curator); - } - - @Test - public void require_that_session_is_initialized() throws Exception { - LocalSession session = createSession(applicationId(), 2); - assertThat(session.getSessionId(), is(2L)); - session = createSession(applicationId(), Long.MAX_VALUE); - assertThat(session.getSessionId(), is(Long.MAX_VALUE)); - assertThat(session.getActiveSessionAtCreate(), is(0L)); - } - - @Test - public void require_that_marking_session_modified_changes_status_to_new() throws Exception { - LocalSession session = createSession(applicationId(), 3); - doPrepare(session, applicationId()); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), Session.Mode.READ); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), Session.Mode.WRITE); - assertThat(session.getStatus(), is(Session.Status.NEW)); - } - - @Test - public void require_that_application_file_can_be_fetched() throws Exception { - LocalSession session = createSession(applicationId(), 3); - ApplicationFile f1 = session.getApplicationFile(Path.fromString("services.xml"), Session.Mode.READ); - ApplicationFile f2 = session.getApplicationFile(Path.fromString("services2.xml"), Session.Mode.READ); - assertTrue(f1.exists()); - assertFalse(f2.exists()); - } - - @Test(expected = IllegalStateException.class) - public void require_that_no_provision_info_throws_exception() throws Exception { - createSession(applicationId(), 3).getAllocatedHosts(); - } - - private LocalSession createSession(ApplicationId applicationId, long sessionId) throws Exception { - return createSession(applicationId, sessionId, Optional.empty()); - } - - private LocalSession createSession(ApplicationId applicationId, long sessionId, - Optional<AllocatedHosts> allocatedHosts) throws Exception { - TenantName tenantName = applicationId.tenant(); - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId, allocatedHosts); - zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), - TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId))); - if (allocatedHosts.isPresent()) { - zkClient.write(allocatedHosts.get()); - } - zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); - TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); - applications.createApplication(applicationId); - LocalSession session = new LocalSession(tenantName, sessionId, FilesApplicationPackage.fromFile(testApp), zkc); - session.setApplicationId(applicationId); - return session; - } - - private void doPrepare(LocalSession session, ApplicationId applicationId) { - doPrepare(session, new PrepareParams.Builder().applicationId(applicationId).build()); - } - - private void doPrepare(LocalSession session, PrepareParams params) { - SessionRepository sessionRepository = tenantRepository.getTenant(params.getApplicationId().tenant()).getSessionRepository(); - sessionRepository.prepareLocalSession(session, getLogger(), params, Optional.empty(), tenantPath, Instant.now()); - } - - private DeployHandlerLogger getLogger() { - return new DeployHandlerLogger(new Slime().get(), false, applicationId()); - } - - private ApplicationId applicationId() { - return new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build(); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index b181ad3e8d6..95cdccf4cb8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -27,7 +28,6 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.TestComponentRegistry; @@ -65,6 +65,7 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -181,7 +182,10 @@ public class SessionPreparerTest { .dryRun(true) .build(), 1); - assertTrue(result.getFileRegistries().get(version321).export().isEmpty()); + Map<Version, FileRegistry> fileRegistries = result.getFileRegistries(); + System.out.println(fileRegistries); + assertEquals(1, fileRegistries.get(version321).export().size()); + assertEquals("./", fileRegistries.get(version321).export().get(0).reference.value()); } @Test @@ -383,8 +387,8 @@ public class SessionPreparerTest { } private DeployHandlerLogger getLogger() { - return new DeployHandlerLogger(new Slime().get(), false /*verbose */, - new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + return DeployHandlerLogger.forApplication( + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build(), false /*verbose */); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java index 8a1a7fbea20..1340935108b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java @@ -4,12 +4,16 @@ package com.yahoo.vespa.config.server.version; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; @@ -20,28 +24,49 @@ import static org.junit.Assert.assertTrue; * @author Ulf Lilleengen */ public class VersionStateTest { + InMemoryFlagSource flagSource = new InMemoryFlagSource(); @Rule public TemporaryFolder tempDir = new TemporaryFolder(); + private final MockCurator curator = new MockCurator(); @Test public void upgrade() throws IOException { + upgrade(true); + upgrade(false); + } + + public void upgrade(boolean distributeApplicationPackage) throws IOException { + flagSource.withBooleanFlag(Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.id(), distributeApplicationPackage); Version unknownVersion = new Version(0, 0, 0); - File versionFile = tempDir.newFile(); - VersionState state = new VersionState(versionFile); + + VersionState state = createVersionState(); assertThat(state.storedVersion(), is(unknownVersion)); assertTrue(state.isUpgraded()); state.saveNewVersion(); assertFalse(state.isUpgraded()); - IOUtils.writeFile(versionFile, "badversion", false); + state.saveNewVersion("badversion"); assertThat(state.storedVersion(), is(unknownVersion)); assertTrue(state.isUpgraded()); - IOUtils.writeFile(versionFile, "5.0.0", false); + state.saveNewVersion("5.0.0"); assertThat(state.storedVersion(), is(new Version(5, 0, 0))); assertTrue(state.isUpgraded()); + // Remove zk node, should find version in ZooKeeper + curator.delete(VersionState.versionPath); + assertThat(state.storedVersion(), is(new Version(5, 0, 0))); + assertTrue(state.isUpgraded()); + + // Save new version, remove version in file, should find version in ZooKeeper + state.saveNewVersion("6.0.0"); + if (distributeApplicationPackage) { + Files.delete(state.versionFile().toPath()); + assertThat(state.storedVersion(), is(new Version(6, 0, 0))); + assertTrue(state.isUpgraded()); + } + state.saveNewVersion(); assertThat(state.currentVersion(), is(state.storedVersion())); assertFalse(state.isUpgraded()); @@ -50,7 +75,9 @@ public class VersionStateTest { @Test public void serverdbfile() throws IOException { File dbDir = tempDir.newFolder(); - VersionState state = new VersionState(new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(dbDir.getAbsolutePath()))); + VersionState state = new VersionState(new ConfigserverConfig.Builder().configServerDBDir(dbDir.getAbsolutePath()).build(), + curator, + new InMemoryFlagSource()); state.saveNewVersion(); File versionFile = new File(dbDir, "vespa_version"); assertTrue(versionFile.exists()); @@ -58,4 +85,8 @@ public class VersionStateTest { assertThat(stored, is(state.currentVersion())); } + private VersionState createVersionState() throws IOException { + return new VersionState(tempDir.newFile(), curator, flagSource); + } + } diff --git a/configutil/src/lib/configstatus.cpp b/configutil/src/lib/configstatus.cpp index 415edf6defc..254fa94a8ec 100644 --- a/configutil/src/lib/configstatus.cpp +++ b/configutil/src/lib/configstatus.cpp @@ -165,7 +165,6 @@ ConfigStatus::action() if (!upToDate) { if (svc.type == "searchnode" || - svc.type == "topleveldispatch" || svc.type == "logd") { std::cerr << "[generation not up-to-date ignored]" << std::endl; diff --git a/configutil/src/tests/model_inspect/model.cfg b/configutil/src/tests/model_inspect/model.cfg index 388bae2256d..bc29226e494 100644 --- a/configutil/src/tests/model_inspect/model.cfg +++ b/configutil/src/tests/model_inspect/model.cfg @@ -115,35 +115,23 @@ hosts[0].services[11].ports[0].number 19106 hosts[0].services[11].ports[0].tags "status admin rpc rtx" hosts[0].services[11].ports[1].number 19107 hosts[0].services[11].ports[1].tags "unused" -hosts[0].services[12].name "topleveldispatch" -hosts[0].services[12].type "topleveldispatch" -hosts[0].services[12].configid "search/cluster.music/tlds/tld.0" -hosts[0].services[12].clustertype "search" -hosts[0].services[12].clustername "music" +hosts[0].services[12].name "docprocservice" +hosts[0].services[12].type "container" +hosts[0].services[12].configid "docproc/cluster.music.indexing/0" +hosts[0].services[12].clustertype "" +hosts[0].services[12].clustername "music.indexing" hosts[0].services[12].index 0 -hosts[0].services[12].ports[0].number 19108 -hosts[0].services[12].ports[0].tags "admin rpc" -hosts[0].services[12].ports[1].number 19109 -hosts[0].services[12].ports[1].tags "fs4" -hosts[0].services[12].ports[2].number 19110 -hosts[0].services[12].ports[2].tags "health json http" -hosts[0].services[13].name "docprocservice" -hosts[0].services[13].type "container" -hosts[0].services[13].configid "docproc/cluster.music.indexing/0" -hosts[0].services[13].clustertype "" -hosts[0].services[13].clustername "music.indexing" -hosts[0].services[13].index 0 -hosts[0].services[13].ports[0].number 19111 -hosts[0].services[13].ports[0].tags "state external query http" -hosts[0].services[13].ports[1].number 19112 -hosts[0].services[13].ports[1].tags "external status http" -hosts[0].services[13].ports[2].number 19113 -hosts[0].services[13].ports[2].tags "messaging rpc" -hosts[0].services[13].ports[3].number 19114 -hosts[0].services[13].ports[3].tags "external fileserver http" -hosts[0].services[13].ports[4].number 19115 -hosts[0].services[13].ports[4].tags "admin rpc" -hosts[0].services[13].ports[5].number 19116 -hosts[0].services[13].ports[5].tags "rmiregistry rmi" -hosts[0].services[13].ports[6].number 19117 -hosts[0].services[13].ports[6].tags "jmx rmi" +hosts[0].services[12].ports[0].number 19111 +hosts[0].services[12].ports[0].tags "state external query http" +hosts[0].services[12].ports[1].number 19112 +hosts[0].services[12].ports[1].tags "external status http" +hosts[0].services[12].ports[2].number 19113 +hosts[0].services[12].ports[2].tags "messaging rpc" +hosts[0].services[12].ports[3].number 19114 +hosts[0].services[12].ports[3].tags "external fileserver http" +hosts[0].services[12].ports[4].number 19115 +hosts[0].services[12].ports[4].tags "admin rpc" +hosts[0].services[12].ports[5].number 19116 +hosts[0].services[12].ports[5].tags "rmiregistry rmi" +hosts[0].services[12].ports[6].number 19117 +hosts[0].services[12].ports[6].tags "jmx rmi" diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 28aeb5155c2..ceaf1426ef2 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -632,7 +632,8 @@ "public final com.yahoo.jdisc.handler.ContentChannel handleRequest(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.ResponseHandler)", "public java.time.Duration getTimeout()", "protected abstract void handleRequest(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.BufferedContentChannel, com.yahoo.jdisc.handler.ResponseHandler)", - "protected void writeErrorResponseOnOverload(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.ResponseHandler)" + "protected void writeErrorResponseOnOverload(com.yahoo.jdisc.Request, com.yahoo.jdisc.handler.ResponseHandler)", + "protected java.util.concurrent.Executor executor()" ], "fields": [ "protected final com.yahoo.jdisc.Metric metric" diff --git a/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java index f87dd3f42d2..6ecb6c75f90 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java @@ -103,9 +103,7 @@ public class ApplicationBundleLoader { // it means that the X-JDisc-Preinstall-Bundle header was used. // However, test osgi frameworks may return multiple bundles when installing a single bundle. if (bundles.size() > 1 && osgi.hasFelixFramework()) { - // TODO: remove if-statement below when the last model with preinstall has rolled out of hosted - if (! bundles.get(0).getSymbolicName().equals("config-model-fat-amended")) - throw new RuntimeException("Bundle '" + bundles.get(0).getSymbolicName() + "' tried to pre-install bundles from disk."); + throw new RuntimeException("Bundle '" + bundles.get(0).getSymbolicName() + "' tried to pre-install bundles from disk."); } reference2Bundle.put(reference, bundles.get(0)); } diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java new file mode 100644 index 00000000000..5e3f873bba9 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core.documentapi; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; + +/** + * Lets a lazily initialised DocumentAccess forwarding to a real MessageBusDocumentAccess be injected in containers. + * + * @author jonmv + */ +public class DocumentAccessProvider extends AbstractComponent implements Provider<VespaDocumentAccess> { + + private final VespaDocumentAccess access; + + @Inject + // TODO jonmv: Have Slobrok and RPC config injected as well. + public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig, + SlobroksConfig slobroksConfig, MessagebusConfig messagebusConfig) { + this.access = new VespaDocumentAccess(documentmanagerConfig, loadTypeConfig, slobroksConfig, messagebusConfig); + } + + @Override + public VespaDocumentAccess get() { + return access; + } + + @Override + public void deconstruct() { + access.shutdown(); + } + + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java new file mode 100644 index 00000000000..2918ffb2c80 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -0,0 +1,102 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core.documentapi; + +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; + +/** + * Wraps a lazily initialised MessageBusDocumentAccess. Lazy to allow it to always be set up. + * Inject this class directly (instead of DocumentAccess) for use in internal code. + * + * @author jonmv + */ +public class VespaDocumentAccess extends DocumentAccess { + + private final MessageBusParams parameters; + private final Object monitor = new Object(); + + private DocumentAccess delegate = null; + private boolean shutDown = false; + + VespaDocumentAccess(DocumentmanagerConfig documentmanagerConfig, + LoadTypeConfig loadTypeConfig, + SlobroksConfig slobroksConfig, + MessagebusConfig messagebusConfig) { + super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); + this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); + this.parameters.setDocumentmanagerConfig(documentmanagerConfig); + this.parameters.getRPCNetworkParams().setSlobroksConfig(slobroksConfig); + this.parameters.getMessageBusParams().setMessageBusConfig(messagebusConfig); + } + + private DocumentAccess delegate() { + synchronized (monitor) { + if (delegate == null) { + if (shutDown) + throw new IllegalStateException("This document access has been shut down"); + + delegate = new MessageBusDocumentAccess(parameters); + } + return delegate; + } + } + + @Override + public void shutdown() { + synchronized (monitor) { + super.shutdown(); + shutDown = true; + if (delegate != null) + delegate.shutdown(); + } + } + + @Override + public SyncSession createSyncSession(SyncParameters parameters) { + return delegate().createSyncSession(parameters); + } + + @Override + public AsyncSession createAsyncSession(AsyncParameters parameters) { + return delegate().createAsyncSession(parameters); + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { + return delegate().createVisitorSession(parameters); + } + + @Override + public VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters) { + return delegate().createVisitorDestinationSession(parameters); + } + + @Override + public SubscriptionSession createSubscription(SubscriptionParameters parameters) { + return delegate().createSubscription(parameters); + } + + @Override + public SubscriptionSession openSubscription(SubscriptionParameters parameters) { + return delegate().openSubscription(parameters); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/package-info.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/package-info.java new file mode 100644 index 00000000000..de524237499 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.core.documentapi; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java index 4d8c245a25a..46b3a86798b 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java @@ -44,8 +44,9 @@ public class DefaultContainerThreadpool extends AbstractComponent implements Aut // get the dreaded thread locals initialized even if they will never run. // That counters what we we want to achieve with the Q that will prefer thread locality. executor.prestartAllCoreThreads(); - threadpool = new ExecutorServiceWrapper(executor, threadPoolMetric, processTerminator, - config.maxThreadExecutionTimeSeconds() * 1000L); + threadpool = new ExecutorServiceWrapper( + executor, threadPoolMetric, processTerminator, config.maxThreadExecutionTimeSeconds() * 1000L, + config.name(), config.queueSize()); } @Override public Executor executor() { return threadpool; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java index 9a73c98597a..771c1da82b6 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java @@ -24,32 +24,35 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { private final ThreadPoolMetric metric; private final ProcessTerminator processTerminator; private final long maxThreadExecutionTimeMillis; + private final int queueCapacity; private final Thread metricReporter; private final AtomicBoolean closed = new AtomicBoolean(false); ExecutorServiceWrapper( WorkerCompletionTimingThreadPoolExecutor wrapped, ThreadPoolMetric metric, ProcessTerminator processTerminator, - long maxThreadExecutionTimeMillis) { + long maxThreadExecutionTimeMillis, String name, int queueCapacity) { this.wrapped = wrapped; this.metric = metric; this.processTerminator = processTerminator; this.maxThreadExecutionTimeMillis = maxThreadExecutionTimeMillis; + this.queueCapacity = queueCapacity; metric.reportThreadPoolSize(wrapped.getPoolSize()); metric.reportActiveThreads(wrapped.getActiveCount()); metricReporter = new Thread(this::reportMetrics); + metricReporter.setName(name + "-threadpool-metric-reporter"); metricReporter.setDaemon(true); metricReporter.start(); } - int queuedTasks() { return wrapped.getQueue().size(); } - - private final void reportMetrics() { + private void reportMetrics() { try { while (!closed.get()) { metric.reportThreadPoolSize(wrapped.getPoolSize()); metric.reportActiveThreads(wrapped.getActiveCount()); + metric.reportWorkQueueSize(wrapped.getQueue().size()); + metric.reportWorkQueueCapacity(queueCapacity); Thread.sleep(100); } } catch (InterruptedException e) { } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java index d9ab020bcb8..18ccf3ba8e5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java @@ -25,6 +25,8 @@ class ThreadPoolMetric { void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); } void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); } void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); } + void reportWorkQueueCapacity(long capacity) { metric.set("jdisc.thread_pool.work_queue.capacity", capacity, defaultContext); } + void reportWorkQueueSize(long size) { metric.set("jdisc.thread_pool.work_queue.size", size, defaultContext); } void reportUnhandledException(Throwable t) { Metric.Context ctx = metric.createContext(Map.of( THREAD_POOL_NAME_DIMENSION, threadPoolName, diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java index 22933556d9f..8243ad07760 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java @@ -121,7 +121,7 @@ public class RequestHandlerTestDriver implements AutoCloseable { } /** - * Read the next piece of data from this channel even it blocking is needed. + * Read the next piece of data from this channel, blocking if needed. * If all data is already read, this returns null. */ public String read() { @@ -147,7 +147,7 @@ public class RequestHandlerTestDriver implements AutoCloseable { return responseString.toString(); } - /** Consumes all <i>currently</i> available data, or return "" if no data is available right now. Never blocks. */ + /** Consumes all <i>currently</i> available data, or returns "" if no data is available right now. Never blocks. */ public String readIfAvailable() { StringBuilder b = new StringBuilder(); while (content.available()>0) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index d729e2371c9..c46488694de 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -119,9 +119,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler metric.add(RENDERING_ERRORS, 1, null); long time = System.currentTimeMillis() - startTime; log.log(time < 900 ? Level.INFO : Level.WARNING, - "IO error while responding to " + " [" - + request.getUri() + "] " + "(total time " - + time + " ms) ", e); + "IO error while responding to [" + request.getUri() + "] (total time " + time + " ms) ", e); try { output.flush(); } catch (Exception ignored) { } } finally { if (channel != null && ! (httpResponse instanceof AsyncHttpResponse)) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 0e8865538ee..ab768dba0d2 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -169,6 +169,8 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { ResponseDispatch.newInstance(Response.Status.SERVICE_UNAVAILABLE).dispatch(responseHandler); } + protected Executor executor() { return executor; } + private class RequestTask implements ResponseHandler, Runnable { final Request request; diff --git a/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java index 2473da3578d..a0f8b58fb6a 100644 --- a/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java @@ -15,7 +15,11 @@ import java.io.OutputStream; */ public class SlimeJsonResponse extends HttpResponse { - private final Slime slime; + protected final Slime slime; + + public SlimeJsonResponse() { + this(new Slime()); + } public SlimeJsonResponse(Slime slime) { super(200); diff --git a/container-di/src/main/resources/configdefinitions/container.bundles.def b/container-di/src/main/resources/configdefinitions/container.bundles.def deleted file mode 100644 index 79e24742398..00000000000 --- a/container-di/src/main/resources/configdefinitions/container.bundles.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container - -# References to both application and platform bundles to install. -bundle[] file diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 26efb6c6312..4f69f5a5b54 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1987,6 +1987,7 @@ "public void <init>(com.yahoo.search.cluster.NodeManager, boolean)", "public void start()", "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", + "public boolean isClosed()", "public void add(java.lang.Object, boolean)", "public com.yahoo.search.cluster.BaseNodeMonitor getNodeMonitor(java.lang.Object)", "public synchronized void failed(java.lang.Object, com.yahoo.search.result.ErrorMessage)", diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 15cf4995b77..27d8bb27ee8 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -62,6 +62,8 @@ public class ClusterMonitor<T> { /** Returns the configuration of this cluster monitor */ public MonitorConfiguration getConfiguration() { return configuration; } + public boolean isClosed() { return closed.get(); } + /** * Adds a new node for monitoring. * The object representing the node must diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index 5e04f1d7a3e..5992d47855f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -86,6 +86,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { @Override public void receive(ResponseOrError<ProtobufResponse> response) { + if (clusterMonitor.isClosed()) return; if (node.isLastReceivedPong(pingSequenceId)) { pongHandler.handle(toPong(response)); } else { diff --git a/container-test/pom.xml b/container-test/pom.xml index 4b5008c8a6f..dfbe5f755ac 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -66,28 +66,5 @@ <artifactId>xercesImpl</artifactId> </dependency> - <!-- These dependencies are necessary in test classpath when using jdisc_http_filters --> - <dependency> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils</artifactId> - <version>1.7.0</version> - <exclusions> - <exclusion> - <!-- To avoid pulling in an older version than what we provide (also affects provided scope). --> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils-core</artifactId> - <version>1.8.0</version> - </dependency> - <dependency> - <groupId>commons-digester</groupId> - <artifactId>commons-digester</artifactId> - <version>1.8</version> - </dependency> </dependencies> </project> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 717a4296b81..ed54060cf5d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -32,6 +32,7 @@ public class DeploymentData { private final Optional<AthenzDomain> athenzDomain; private final Optional<ApplicationRoles> applicationRoles; private final Optional<Quota> quota; + private final boolean internalRestart; public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, @@ -39,7 +40,8 @@ public class DeploymentData { Optional<DockerImage> dockerImageRepo, Optional<AthenzDomain> athenzDomain, Optional<ApplicationRoles> applicationRoles, - Optional<Quota> quota) { + Optional<Quota> quota, + boolean internalRestart) { this.instance = requireNonNull(instance); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); @@ -50,6 +52,7 @@ public class DeploymentData { this.athenzDomain = athenzDomain; this.applicationRoles = applicationRoles; this.quota = quota; + this.internalRestart = internalRestart; } public ApplicationId instance() { @@ -91,4 +94,8 @@ public class DeploymentData { public Optional<Quota> quota() { return quota; } + + public boolean internalRestart() { + return internalRestart; + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java index cae768afc90..4e1efc1b6d9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Quota.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.billing; import java.util.Objects; @@ -5,8 +6,13 @@ import java.util.Optional; /** * Quota information transmitted to the configserver on deploy. + * + * @author andreer + * @author ogronnesby */ public class Quota { + private static final Quota UNLIMITED = new Quota(Optional.empty(), Optional.empty()); + private static final Quota ZERO = new Quota(0, 0); private final Optional<Integer> maxClusterSize; private final Optional<Integer> budget; // in USD/hr, as calculated by NodeResources @@ -40,6 +46,14 @@ public class Quota { return new Quota(maxClusterSize, Optional.of(budget)); } + public static Quota zero() { + return ZERO; + } + + public static Quota unlimited() { + return UNLIMITED; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java index 80f09688ec8..bb0613428f5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/NodeEntity.java @@ -14,11 +14,13 @@ public class NodeEntity { private final String hostname; private final Optional<String> model; private final Optional<String> manufacturer; + private final Optional<String> switchHostname; - public NodeEntity(String hostname, String model, String manufacturer) { + public NodeEntity(String hostname, String model, String manufacturer, String switchHostname) { this.hostname = Objects.requireNonNull(hostname); this.model = nonBlank(model); this.manufacturer = nonBlank(manufacturer); + this.switchHostname = nonBlank(switchHostname); } public String hostname() { @@ -35,6 +37,11 @@ public class NodeEntity { return manufacturer; } + /** The hostname of network switch this node is connected to */ + public Optional<String> switchHostname() { + return switchHostname; + } + private static Optional<String> nonBlank(String s) { return Optional.ofNullable(s).filter(v -> !v.isBlank()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index e6310cc6432..a00992da815 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -45,6 +45,11 @@ import static com.yahoo.yolean.Exceptions.uncheck; * The flag files must reside in a 'flags/' root directory containing a directory for each flag name: * {@code ./flags/<flag-id>/*.json} * + * Optionally, there can be an arbitrary number of directories "between" 'flags/' root directory and + * the flag name directory: + * {@code ./flags/onelevel/<flag-id>/*.json} + * {@code ./flags/onelevel/anotherlevel/<flag-id>/*.json} + * * @author bjorncs */ public class SystemFlagsDataArchive { @@ -155,7 +160,7 @@ public class SystemFlagsDataArchive { if (!filename.endsWith(".json")) { throw new IllegalArgumentException(String.format("Only JSON files are allowed in 'flags/' directory (found '%s')", filePath.toString())); } - FlagId directoryDeducedFlagId = new FlagId(filePath.getName(1).toString()); + FlagId directoryDeducedFlagId = new FlagId(filePath.getName(filePath.getNameCount()-2).toString()); FlagData flagData; if (rawData.isBlank()) { flagData = new FlagData(directoryDeducedFlagId); @@ -178,6 +183,13 @@ public class SystemFlagsDataArchive { "\nSee https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax"); } } + + if (builder.hasFile(filename, flagData)) { + throw new IllegalArgumentException( + String.format("Flag data file in '%s' contains redundant flag data for id '%s' already set in another directory!", + filePath, flagData.id())); + } + builder.addFile(filename, flagData); } @@ -236,6 +248,10 @@ public class SystemFlagsDataArchive { return this; } + public boolean hasFile(String filename, FlagData data) { + return files.containsKey(data.id()) && files.get(data.id()).containsKey(filename); + } + public SystemFlagsDataArchive build() { Map<FlagId, Map<String, FlagData>> copy = new TreeMap<>(); files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index 4cdbe5241bc..aca991ec637 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -84,6 +84,19 @@ public class SystemFlagsDataArchiveTest { } @Test + public void supports_multi_level_flags_directory() { + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level/")); + assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "default"); + } + + @Test + public void duplicated_flagdata_is_detected() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("contains redundant flag data for id 'my-test-flag' already set in another directory!"); + var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level-with-duplicated-flagdata/")); + } + + @Test public void empty_files_are_handled_as_no_flag_data_for_target() { var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); assertNoFlagData(archive, FLAG_WITH_EMPTY_DATA, mainControllerTarget); diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-1/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +}
\ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level-with-duplicated-flagdata/flags/group-2/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +}
\ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level/flags/group-1/my-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +}
\ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json b/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json new file mode 100644 index 00000000000..e30485b755c --- /dev/null +++ b/controller-api/src/test/resources/system-flags-multi-level/flags/group-2/my-other-test-flag/default.json @@ -0,0 +1,8 @@ +{ + "id" : "my-other-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7135cbe77c9..a6c177230d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -118,6 +118,7 @@ public class ApplicationController { private final EndpointCertificateManager endpointCertificateManager; private final StringFlag dockerImageRepoFlag; private final BooleanFlag provisionApplicationRoles; + private final BooleanFlag deployWithInternalRestart; private final BillingController billingController; ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock, @@ -132,6 +133,7 @@ public class ApplicationController { this.applicationStore = controller.serviceRegistry().applicationStore(); this.dockerImageRepoFlag = Flags.DOCKER_IMAGE_REPO.bindTo(flagSource); this.provisionApplicationRoles = Flags.PROVISION_APPLICATION_ROLES.bindTo(flagSource); + this.deployWithInternalRestart = Flags.DEPLOY_WITH_INTERNAL_RESTART.bindTo(controller.flagSource()); this.billingController = billingController; deploymentTrigger = new DeploymentTrigger(controller, clock); @@ -521,6 +523,11 @@ public class ApplicationController { .filter(s -> !s.isBlank()) .map(DockerImage::fromString); + boolean internalRestart = deployWithInternalRestart + .with(FetchVector.Dimension.ZONE_ID, zone.value()) + .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) + .value(); + Optional<AthenzDomain> domain = controller.tenants().get(application.tenant()) .filter(tenant-> tenant instanceof AthenzTenant) .map(tenant -> ((AthenzTenant)tenant).domain()); @@ -534,7 +541,8 @@ public class ApplicationController { ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, - endpoints, endpointCertificateMetadata, dockerImageRepo, domain, applicationRoles, quota)); + endpoints, endpointCertificateMetadata, dockerImageRepo, domain, + applicationRoles, quota, internalRestart)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 7dd6126dabc..91be3a4df21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -65,7 +65,7 @@ public enum SystemApplication { .orElse(false); } - /** Returns whether this should receive OS upgrades in given cloud */ + /** Returns whether this should receive OS upgrades in given zone */ public boolean shouldUpgradeOsIn(ZoneId zone, Controller controller) { if (controller.zoneRegistry().zones().reprovisionToUpgradeOs().ids().contains(zone)) { return nodeType == NodeType.host; // TODO(mpolden): Remove once all node types are supported diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index d39e0c134fd..709064c8715 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -107,7 +107,9 @@ public class DeploymentTrigger { Map<JobId, List<Versions>> newJobsToRun = jobs.deploymentStatus(application.get()).jobsToRun(); for (Run run : jobs.active(application.get().id().instance(instanceName))) { if ( ! run.id().type().environment().isManuallyDeployed() - && ! newJobsToRun.getOrDefault(run.id().job(), List.of()).contains(run.versions())) + && newJobsToRun.getOrDefault(run.id().job(), List.of()).stream() + .noneMatch(versions -> versions.targetsMatch(run.versions()) + && versions.sourcesMatchIfPresent(run.versions()))) jobs.abort(run.id()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 3ebc8240889..af9acc7c3a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -987,7 +987,7 @@ public class InternalStepRunner implements StepRunner { Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 120); } + Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 240); } Duration testerCertificate() { return Duration.ofMinutes(300); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index 7a0349d7737..a05b3425a01 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -58,9 +58,9 @@ public class Versions { /** Returns whether source versions are present and match those of the given job other versions. */ public boolean sourcesMatchIfPresent(Versions versions) { - return (sourcePlatform.filter(version -> ! version.equals(targetPlatform)).isEmpty() || + return (sourcePlatform.map(targetPlatform::equals).orElse(true) || sourcePlatform.equals(versions.sourcePlatform())) && - (sourceApplication.filter(version -> ! version.equals(targetApplication)).isEmpty() || + (sourceApplication.map(targetApplication::equals).orElse(true) || sourceApplication.equals(versions.sourceApplication())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index e0f2f0718ef..e5ee06f81dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -74,7 +74,7 @@ public class JobRunner extends ControllerMaintainer { /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */ public void advance(Run run) { if ( ! run.hasFailed() - && run.start().isBefore(controller().clock().instant().minus(jobTimeout))) { + && controller().clock().instant().isAfter(run.start().plus(jobTimeout))) { jobs.abort(run.id()); advance(jobs.run(run.id()).get()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index ba40f9c2085..114a2967e9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -42,6 +42,7 @@ import java.util.stream.Collectors; */ public class RoutingApiHandler extends AuditLoggingRequestHandler { + private static final String OPTIONAL_PREFIX = "/api"; private final Controller controller; public RoutingApiHandler(Context ctx, Controller controller) { @@ -52,7 +53,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { @Override public HttpResponse auditAndHandle(HttpRequest request) { try { - var path = new Path(request.getUri()); + var path = new Path(request.getUri(), OPTIONAL_PREFIX); switch (request.getMethod()) { case GET: return get(path, request); case POST: return post(path); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 7c44ae6d1a5..4a245fb3555 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -11,6 +11,9 @@ import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -33,12 +36,14 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { private final ZoneRegistry zoneRegistry; private final ConfigServerRestExecutor proxy; + private final FlagSource flagSource; public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry, ConfigServerRestExecutor proxy, Controller controller) { super(parentCtx, controller.auditLogger()); this.zoneRegistry = serviceRegistry.zoneRegistry(); this.proxy = proxy; + this.flagSource = controller.flagSource(); } @Override @@ -108,11 +113,12 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { } private ProxyRequest proxyRequest(ZoneId zoneId, String path, HttpRequest request) { - // TODO: Use config server VIP for all zones that have one - if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) { - return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); - } - return ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); + boolean useConfigServerVip = Flags.USE_CONFIG_SERVER_VIP.bindTo(flagSource) + .with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value(); + + return useConfigServerVip + ? ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request) + : ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 942c9ac037b..f3b4d1f6457 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -185,10 +185,9 @@ public class ControllerTest { assertNull("Deployment job was removed", context.instanceJobs().get(productionUsWest1)); // Submission has stored application meta. - assertArrayEquals(applicationPackage.metaDataZip(), - tester.controllerTester().serviceRegistry().applicationStore() - .getMeta(context.instanceId()) - .get(tester.clock().instant())); + assertNotNull(tester.controllerTester().serviceRegistry().applicationStore() + .getMeta(context.instanceId()) + .get(tester.clock().instant())); // Meta data tombstone placed on delete tester.clock().advance(Duration.ofSeconds(1)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index cc92c6cd271..522d44ed667 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -116,7 +116,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 4, tester.jobs().active().size()); canary0.runJob(systemTest); - canary0.timeOutUpgrade(stagingTest); + canary0.failDeployment(stagingTest); tester.controllerTester().computeVersionStatus(); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); @@ -153,7 +153,7 @@ public class UpgraderTest { assertEquals("Canaries done: Should upgrade defaults", 6, tester.jobs().active().size()); default0.runJob(systemTest); - default0.timeOutConvergence(stagingTest); + default0.failDeployment(stagingTest); default1.deployPlatform(version3); default2.deployPlatform(version3); @@ -1035,7 +1035,7 @@ public class UpgraderTest { assertEquals(v3, application.instanceJobs().get(stagingTest).lastSuccess().get().versions().targetPlatform()); // First deployment fails and then successfully upgrades to v3 - application.timeOutUpgrade(productionUsCentral1); + application.failDeployment(productionUsCentral1); application.runJob(productionUsCentral1); // Deployments are now on 3 versions diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index f79051f2edb..4b4a8415d69 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -106,6 +106,7 @@ public class ControllerContainerTest { " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.routing.RoutingApiHandler'>\n" + " <binding>http://*/routing/v1/*</binding>\n" + + " <binding>http://*/api/routing/v1/*</binding>\n" + " </handler>\n" + variablePartXml() + "</container>"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 3d583f12a1c..182af3e47ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -88,7 +88,7 @@ public class JobControllerApiHandlerHelperTest { assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); tester.runner().run(); - tester.clock().advance(Duration.ofHours(2).plusSeconds(1)); + tester.clock().advance(Duration.ofHours(4).plusSeconds(1)); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index e45bd190d5f..b8c48eb3d0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -128,8 +128,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/system-test/run/3", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -320,9 +320,9 @@ "dependencies": [], "declared": true, "instance": "default", - "readyAt": 7953000, - "delayedUntil": 7953000, - "coolingDownUntil": 7953000, + "readyAt": 15153000, + "delayedUntil": 15153000, + "coolingDownUntil": 15153000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -351,8 +351,8 @@ { "id": 5, "url": "https://some.url:43/instance/default/job/staging-test/run/5", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -432,8 +432,8 @@ { "id": 4, "url": "https://some.url:43/instance/default/job/staging-test/run/4", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", @@ -513,8 +513,8 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/staging-test/run/3", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "status": "success", "versions": { "targetPlatform": "6.1.0", @@ -756,7 +756,7 @@ ], "declared": true, "instance": "default", - "readyAt": 7203000, + "readyAt": 14403000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -773,7 +773,7 @@ { "id": 3, "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", - "start": 7203000, + "start": 14403000, "status": "running", "versions": { "targetPlatform": "6.1.0", @@ -1050,7 +1050,7 @@ "id": 2, "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, - "end": 7202000, + "end": 14402000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index 2053b5a80b1..3a78f8c44a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -2,8 +2,8 @@ "1": { "id": 1, "status": "success", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index 285e7f14e8f..2601937faee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -8,8 +8,8 @@ "targetApplication": {}, "targetPlatform": "7.1.0" }, - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "id": 1, "steps": [ { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 2ad35968732..37ae9e4b56b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -94,8 +94,8 @@ "3": { "id": 3, "status": "success", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -146,8 +146,8 @@ "4": { "id": 4, "status": "installationFailed", - "start": 7203000, - "end": 7203000, + "start": 14403000, + "end": 14403000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -194,8 +194,8 @@ "5": { "id": 5, "status": "installationFailed", - "start": 7303000, - "end": 7303000, + "start": 14503000, + "end": 14503000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index a941b824b53..e8c8e57aa9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -4,127 +4,127 @@ "log": { "deployTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "No services requiring restart." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment successful." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "foo" } ], "installTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "No services requiring restart." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment successful." }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "foo" } ], "installInitialReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "######## Details for all nodes ########" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- platform dockerImage:6.1" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deployment expired before installation was successful." } ], "deactivateReal": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..." } ], "deactivateTester": [ { - "at": 7303000, + "at": 14503000, "type": "info", "message": "Deactivating tester of tenant.application in staging.us-east-3 ..." } @@ -134,19 +134,19 @@ "steps": { "deployTester": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "installTester": { "status": "unfinished", - "startMillis": 7303000 + "startMillis": 14503000 }, "deployInitialReal": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "installInitialReal": { "status": "failed", - "startMillis": 7303000, + "startMillis": 14503000, "convergence": { "nodes": 1, "down": 0, @@ -182,19 +182,19 @@ }, "copyVespaLogs": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "deactivateReal": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "deactivateTester": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 }, "report": { "status": "succeeded", - "startMillis": 7303000 + "startMillis": 14503000 } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index fefd23eb67c..b0549662ab0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -76,6 +76,12 @@ public class RoutingApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "", Request.Method.GET), new File("discovery/environment.json")); + + // GET instance with api prefix (test that the /api prefix works) + tester.assertResponse(authenticatedRequest("http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/", + "", + Request.Method.GET), + new File("discovery/instance_api.json")); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json new file mode 100644 index 00000000000..a9e789d9fe9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/" + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index a7adac7f89d..61915860d7c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -81,7 +81,7 @@ public class ZoneApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/zone/v2/dev/aws-us-north-2/nodes/v2/node/node1", "{\"currentRestartGeneration\": 1}", Method.PATCH), "ok"); - assertLastRequest(ZoneId.from("dev", "aws-us-north-2"), 1, "PATCH"); + assertLastRequest(ZoneId.from("dev", "aws-us-north-2"), 2, "PATCH"); assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get()); assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty()); diff --git a/document/src/main/java/com/yahoo/document/DocumentUtil.java b/document/src/main/java/com/yahoo/document/DocumentUtil.java index 6e781ce9011..78fe748dac8 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUtil.java +++ b/document/src/main/java/com/yahoo/document/DocumentUtil.java @@ -3,7 +3,7 @@ package com.yahoo.document; /** * Class containing static utility function related to documents. - * @author einarmr + * @author Einar M Rosenvinge * @since 5.1.9 */ public class DocumentUtil { diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java index 5da6b58ac98..3b275d571e0 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java @@ -23,7 +23,7 @@ import com.yahoo.vespaxmlparser.RemoveFeedOperation; * The feed reader will take ownership of the input stream and close it when the * last parseable document has been read. * - * @author steinar + * @author Steinar Knutsen */ public class JsonFeedReader implements FeedReader { diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index a28ce1ee6db..f5f2a7c1845 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -40,13 +40,17 @@ "public com.yahoo.documentapi.Result put(com.yahoo.document.Document, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract double getCurrentWindowSize()" ], "fields": [] @@ -73,6 +77,7 @@ ], "methods": [ "public static com.yahoo.documentapi.DocumentAccess createDefault()", + "public static com.yahoo.documentapi.DocumentAccess createForNonContainer()", "protected void <init>(com.yahoo.documentapi.DocumentAccessParams)", "public abstract com.yahoo.documentapi.SyncSession createSyncSession(com.yahoo.documentapi.SyncParameters)", "public abstract com.yahoo.documentapi.AsyncSession createAsyncSession(com.yahoo.documentapi.AsyncParameters)", @@ -129,6 +134,7 @@ "public void <init>(long, java.lang.String, boolean)", "public void <init>(long, com.yahoo.document.DocumentId, java.lang.String, boolean)", "public void <init>(long, com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void <init>(long, com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.DocumentId getDocumentId()", "public int hashCode()", "public boolean equals(java.lang.Object)", @@ -148,6 +154,26 @@ ], "fields": [] }, + "com.yahoo.documentapi.DocumentOperationParameters": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public static com.yahoo.documentapi.DocumentOperationParameters parameters()", + "public com.yahoo.documentapi.DocumentOperationParameters withPriority(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.DocumentOperationParameters withFieldSet(com.yahoo.document.fieldset.FieldSet)", + "public com.yahoo.documentapi.DocumentOperationParameters withFieldSet(java.lang.String)", + "public com.yahoo.documentapi.DocumentOperationParameters withRoute(java.lang.String)", + "public com.yahoo.documentapi.DocumentOperationParameters withTraceLevel(int)", + "public java.util.Optional priority()", + "public java.util.Optional fieldSet()", + "public java.util.Optional route()", + "public java.util.OptionalInt traceLevel()" + ], + "fields": [] + }, "com.yahoo.documentapi.DocumentResponse": { "superClass": "com.yahoo.documentapi.Response", "interfaces": [], @@ -157,11 +183,13 @@ "methods": [ "public void <init>(long)", "public void <init>(long, com.yahoo.document.Document)", + "public void <init>(long, com.yahoo.document.Document, com.yahoo.messagebus.Trace)", "public void <init>(long, java.lang.String, boolean)", - "public void <init>(long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void <init>(long, com.yahoo.document.Document, java.lang.String, boolean)", "public void <init>(long, com.yahoo.document.Document, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void <init>(long, com.yahoo.document.Document, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.Document getDocument()", + "public boolean isSuccess()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" @@ -181,6 +209,7 @@ "public void <init>(long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void <init>(long, com.yahoo.document.DocumentUpdate, java.lang.String, boolean)", "public void <init>(long, com.yahoo.document.DocumentUpdate, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void <init>(long, com.yahoo.document.DocumentUpdate, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.DocumentUpdate getDocumentUpdate()", "public int hashCode()", "public boolean equals(java.lang.Object)", @@ -353,7 +382,9 @@ ], "methods": [ "public void <init>(long, boolean)", + "public void <init>(long, boolean, com.yahoo.messagebus.Trace)", "public boolean wasFound()", + "public boolean isSuccess()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" @@ -390,10 +421,12 @@ "public void <init>(long, java.lang.String)", "public void <init>(long, java.lang.String, boolean)", "public void <init>(long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", + "public void <init>(long, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public java.lang.String getTextMessage()", "public boolean isSuccess()", "public com.yahoo.documentapi.Response$Outcome outcome()", "public long getRequestId()", + "public com.yahoo.messagebus.Trace getTrace()", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -541,14 +574,18 @@ "methods": [ "public abstract void put(com.yahoo.document.DocumentPut)", "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters, java.time.Duration)", "public abstract boolean remove(com.yahoo.document.DocumentRemove)", "public abstract boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract boolean update(com.yahoo.document.DocumentUpdate)", - "public abstract boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)" + "public abstract boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)" ], "fields": [] }, @@ -560,7 +597,9 @@ ], "methods": [ "public void <init>(long, boolean)", + "public void <init>(long, boolean, com.yahoo.messagebus.Trace)", "public boolean wasFound()", + "public boolean isSuccess()", "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()" @@ -1019,13 +1058,17 @@ "methods": [ "public com.yahoo.documentapi.Result put(com.yahoo.document.Document)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result send(com.yahoo.messagebus.Message)", "public com.yahoo.documentapi.Response getNext()", "public com.yahoo.documentapi.Response getNext(int)", @@ -1129,14 +1172,16 @@ "public com.yahoo.messagebus.Reply syncSend(com.yahoo.messagebus.Message)", "public void put(com.yahoo.document.DocumentPut)", "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters, java.time.Duration)", "public boolean remove(com.yahoo.document.DocumentRemove)", "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", "public boolean update(com.yahoo.document.DocumentUpdate)", "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public java.lang.String getRoute()", "public void setRoute(java.lang.String)", "public int getTraceLevel()", diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java index 9f4ceaad37f..60f70a91338 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java @@ -7,6 +7,8 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentUpdate; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; + /** * <p>A session for asynchronous access to a document repository. * This class provides document repository writes and random access with high @@ -42,10 +44,11 @@ public interface AsyncSession extends Session { * If it was not a success, this method has no further effects.</p> * * @param document the Document to put + * @param priority the priority with which to send the operation * @return the synchronous result of this operation */ default Result put(Document document, DocumentProtocol.Priority priority) { - return put(new DocumentPut(document), priority); + return put(new DocumentPut(document), parameters().withPriority(priority)); } /** @@ -60,7 +63,7 @@ public interface AsyncSession extends Session { * @return the synchronous result of this operation */ default Result put(DocumentPut documentPut) { - return put(documentPut, DocumentProtocol.Priority.NORMAL_3); + return put(documentPut, parameters()); } /** @@ -72,10 +75,26 @@ public interface AsyncSession extends Session { * If it was not a success, this method has no further effects.</p> * * @param documentPut the DocumentPut to perform + * @param priority the priority with which to send the operation * @return the synchronous result of this operation */ - // TODO Vespa 8: Make this the one to implement. default Result put(DocumentPut documentPut, DocumentProtocol.Priority priority) { + return put(documentPut, parameters().withPriority(priority)); + } + + /** + * <p>Puts a document, with optional conditions on the operation. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will either be a success, or contain the document submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param documentPut the DocumentPut to perform + * @param parameters parameters for the operation + * @return the synchronous result of this operation + */ + default Result put(DocumentPut documentPut, DocumentOperationParameters parameters) { return put(documentPut.getDocument()); } @@ -126,6 +145,23 @@ public interface AsyncSession extends Session { * @throws UnsupportedOperationException if this access implementation does not support retrieving */ default Result get(DocumentId id, DocumentProtocol.Priority priority) { + return get(id, parameters().withPriority(priority)); + } + + /** + * <p>Gets a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will contain the requested document if it is a success. + * If it was not a success, this method has no further effects.</p> + * + * @param id the id of the document to get + * @param parameters parameters for the operation + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support retrieving + */ + default Result get(DocumentId id, DocumentOperationParameters parameters) { return get(id); } @@ -158,6 +194,23 @@ public interface AsyncSession extends Session { * @throws UnsupportedOperationException if this access implementation does not support removal */ default Result remove(DocumentId id, DocumentProtocol.Priority priority) { + return remove(id, parameters().withPriority(priority)); + } + + /** + * <p>Removes a document if it is present. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentIdResponse} objects to appear within the timeout time of this session. + * The response returned later will either be a success, or contain the document id submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param id the id of the document to remove + * @param parameters parameters for the operation + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support removal + */ + default Result remove(DocumentId id, DocumentOperationParameters parameters) { return remove(id); } @@ -189,6 +242,23 @@ public interface AsyncSession extends Session { * @throws UnsupportedOperationException if this access implementation does not support update */ default Result update(DocumentUpdate update, DocumentProtocol.Priority priority) { + return update(update, parameters().withPriority(priority)); + } + + /** + * <p>Updates a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentUpdateResponse} within the timeout time of this session. + * The returned response returned later will either be a success or contain the update submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param update the updates to perform + * @param parameters parameters for the operation + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support update + */ + default Result update(DocumentUpdate update, DocumentOperationParameters parameters) { return update(update); } @@ -199,4 +269,5 @@ public interface AsyncSession extends Session { */ double getCurrentWindowSize(); + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java index 308eafcd596..1aa5c4c0df0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java @@ -5,6 +5,7 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.select.parser.ParseException; import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; /** * <p>This is the starting point of the <b>document api</b>. This api provides @@ -27,9 +28,9 @@ import com.yahoo.config.subscription.ConfigSubscriber; * <p>This class is the factory for creating the four session types mentioned above.</p> * * <p>There may be multiple implementations of the document api classes. If - * default configuration is sufficient, use the {@link #createDefault} method to - * return a running document access. Note that there are running threads within - * an access object, so you must shut it down when done.</p> + * default configuration is sufficient, simply inject a {@code DocumentAccess} to + * obtain a running document access. If you instead create a concrete implementation, note that + * there are running threads within an access object, so you must shut it down when done.</p> * * <p>An implementation of the Document Api may support just a subset of the * access types defined in this interface. For example, some document @@ -55,13 +56,29 @@ public abstract class DocumentAccess { * while attempting to create such an object, this method will throw an * exception. * - * @deprecated Inject a DocumentManagerConfig and create a MessageBusDocumentAccess from this instead. + * @deprecated DocumentAccess may be injected in containers — otherwise use {@link #createForNonContainer()}. * * @return a running document access object with all default configuration */ @Deprecated(since = "7") public static DocumentAccess createDefault() { - return new com.yahoo.documentapi.messagebus.MessageBusDocumentAccess(); + return new MessageBusDocumentAccess(); + } + + + /** + * This is a convenience method to return a document access object when running + * outside of a Vespa application container, with all default parameter values. + * The client that calls this method is also responsible for shutting the object + * down when done. If an error occurred while attempting to create such an object, + * this method will throw an exception. + * This document access requires new config subscriptions to be set up, which should + * be avoided in application containers, but is suitable for, e.g., CLIs. + * + * @return a running document access object with all default configuration + */ + public static DocumentAccess createForNonContainer() { + return new MessageBusDocumentAccess(); } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java index 34ab47571cf..e4a44fb88cd 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java @@ -2,6 +2,7 @@ package com.yahoo.documentapi; import com.yahoo.document.DocumentId; +import com.yahoo.messagebus.Trace; /** * The asynchronous response to a document remove operation. @@ -61,7 +62,19 @@ public class DocumentIdResponse extends Response { * @param outcome the outcome of the operation */ public DocumentIdResponse(long requestId, DocumentId documentId, String textMessage, Outcome outcome) { - super(requestId, textMessage, outcome); + this(requestId, documentId, textMessage, outcome, null); + } + + + /** + * Creates a response containing a textual message and/or a document id + * + * @param documentId the DocumentId to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of the operation + */ + public DocumentIdResponse(long requestId, DocumentId documentId, String textMessage, Outcome outcome, Trace trace) { + super(requestId, textMessage, outcome, null); this.documentId = documentId; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java new file mode 100644 index 00000000000..3258c2f5b2c --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java @@ -0,0 +1,71 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.fieldset.FieldSet; +import com.yahoo.document.fieldset.FieldSetRepo; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +import java.util.Optional; +import java.util.OptionalInt; + +import static java.util.Objects.requireNonNull; + +/** + * Optional parameters for a document operation. Immutable class. + * + * @author jonmv + */ +public class DocumentOperationParameters { + + private static final DocumentOperationParameters empty = new DocumentOperationParameters(null, null, null, -1); + + private final DocumentProtocol.Priority priority; + private final String fieldSet; + private final String route; + private final int traceLevel; + + private DocumentOperationParameters(DocumentProtocol.Priority priority, String fieldSet, String route, int traceLevel) { + this.priority = priority; + this.fieldSet = fieldSet; + this.route = route; + this.traceLevel = traceLevel; + } + + public static DocumentOperationParameters parameters() { + return empty; + } + + /** Sets the priority with which to perform an operation. */ + public DocumentOperationParameters withPriority(DocumentProtocol.Priority priority) { + return new DocumentOperationParameters(requireNonNull(priority), fieldSet, route, traceLevel); + } + + /** Sets the field set used for retrieval. */ + public DocumentOperationParameters withFieldSet(FieldSet fieldSet) { + return new DocumentOperationParameters(priority, new FieldSetRepo().serialize(fieldSet), route, traceLevel); + } + + /** Sets the field set used for retrieval. */ + public DocumentOperationParameters withFieldSet(String fieldSet) { + return new DocumentOperationParameters(priority, requireNonNull(fieldSet), route, traceLevel); + } + + /** Sets the route along which to send the operation. */ + public DocumentOperationParameters withRoute(String route) { + return new DocumentOperationParameters(priority, fieldSet, requireNonNull(route), traceLevel); + } + + /** Sets the trace level for an operation. */ + public DocumentOperationParameters withTraceLevel(int traceLevel) { + if (traceLevel < 0 || traceLevel > 9) + throw new IllegalArgumentException("Trace level must be from 0 (no tracing) to 9 (maximum)"); + + return new DocumentOperationParameters(priority, fieldSet, route, traceLevel); + } + + public Optional<DocumentProtocol.Priority> priority() { return Optional.ofNullable(priority); } + public Optional<String> fieldSet() { return Optional.ofNullable(fieldSet); } + public Optional<String> route() { return Optional.ofNullable(route); } + public OptionalInt traceLevel() { return traceLevel >= 0 ? OptionalInt.of(traceLevel) : OptionalInt.empty(); } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java index 172e5fd11c0..2a02b70d4fc 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -2,6 +2,7 @@ package com.yahoo.documentapi; import com.yahoo.document.Document; +import com.yahoo.messagebus.Trace; /** * The asynchronous response to a document put or get operation. @@ -25,30 +26,28 @@ public class DocumentResponse extends Response { * @param document the Document to encapsulate in the Response */ public DocumentResponse(long requestId, Document document) { - super(requestId); - this.document = document; + this(requestId, document, null); } /** - * Creates a response containing a textual message + * Creates a successful response containing a document * - * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call + * @param document the Document to encapsulate in the Response */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public DocumentResponse(long requestId, String textMessage, boolean success) { - super(requestId, textMessage, success); - document = null; + public DocumentResponse(long requestId, Document document, Trace trace) { + this(requestId, document, null, document != null ? Outcome.SUCCESS : Outcome.NOT_FOUND, trace); } /** * Creates a response containing a textual message * * @param textMessage the message to encapsulate in the Response - * @param outcome the outcome of this operation + * @param success true if the response represents a successful call */ - public DocumentResponse(long requestId, String textMessage, Outcome outcome) { - this(requestId, null, textMessage, outcome); + @Deprecated(since = "7") // TODO: Remove on Vespa 8 + public DocumentResponse(long requestId, String textMessage, boolean success) { + super(requestId, textMessage, success ? Outcome.NOT_FOUND : Outcome.ERROR); + document = null; } /** @@ -72,7 +71,19 @@ public class DocumentResponse extends Response { * @param outcome the outcome of this operation */ public DocumentResponse(long requestId, Document document, String textMessage, Outcome outcome) { - super(requestId, textMessage, outcome); + this(requestId, document, textMessage, outcome, null); + } + + + /** + * Creates a response containing a textual message and/or a document + * + * @param document the Document to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of this operation + */ + public DocumentResponse(long requestId, Document document, String textMessage, Outcome outcome, Trace trace) { + super(requestId, textMessage, outcome, trace); this.document = document; } @@ -84,6 +95,12 @@ public class DocumentResponse extends Response { */ public Document getDocument() { return document; } + @Override + public boolean isSuccess() { + // TODO: is it right that Get operations are successful without a result, in this API? + return super.isSuccess() || outcome() == Outcome.NOT_FOUND; + } + public int hashCode() { return super.hashCode() + (document == null ? 0 : document.hashCode()); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java index 3294c216d96..d34873aeaa6 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java @@ -2,6 +2,7 @@ package com.yahoo.documentapi; import com.yahoo.document.DocumentUpdate; +import com.yahoo.messagebus.Trace; /** * The asynchronous response to a document update operation. @@ -71,7 +72,19 @@ public class DocumentUpdateResponse extends Response { * @param outcome the outcome of this operation */ public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate, String textMessage, Outcome outcome) { - super(requestId, textMessage, outcome); + this(requestId, documentUpdate, textMessage, outcome, null); + } + + + /** + * Creates a response containing a textual message and/or a document update + * + * @param documentUpdate the DocumentUpdate to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of this operation + */ + public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate, String textMessage, Outcome outcome, Trace trace) { + super(requestId, textMessage, outcome, trace); this.documentUpdate = documentUpdate; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java index 502588a3d5f..2d3f2934890 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import com.yahoo.messagebus.Trace; + /** * This response is provided for successful document remove operations. Use the * wasFound() method to check whether or not the document was actually found. @@ -12,7 +14,11 @@ public class RemoveResponse extends Response { private final boolean wasFound; public RemoveResponse(long requestId, boolean wasFound) { - super(requestId); + this(requestId, wasFound, null); + } + + public RemoveResponse(long requestId, boolean wasFound, Trace trace) { + super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND, trace); this.wasFound = wasFound; } @@ -21,6 +27,10 @@ public class RemoveResponse extends Response { } @Override + // TODO: fix this when/if NOT_FOUND is no longer a success. + public boolean isSuccess() { return super.isSuccess() || outcome() == Outcome.NOT_FOUND; } + + @Override public int hashCode() { return super.hashCode() + Boolean.valueOf(wasFound).hashCode(); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java index 4c95a648949..0a541a92c6a 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Response.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import com.yahoo.messagebus.Trace; + import java.util.Objects; import static com.yahoo.documentapi.Response.Outcome.ERROR; @@ -19,6 +21,7 @@ public class Response { private final long requestId; private final String textMessage; private final Outcome outcome; + private final Trace trace; /** Creates a successful response containing no information */ public Response(long requestId) { @@ -52,9 +55,20 @@ public class Response { * @param outcome the outcome of the operation */ public Response(long requestId, String textMessage, Outcome outcome) { + this(requestId, textMessage, outcome, null); + } + + /** + * Creates a response containing a textual message + * + * @param textMessage the message to encapsulate in the Response + * @param outcome the outcome of the operation + */ + public Response(long requestId, String textMessage, Outcome outcome, Trace trace) { this.requestId = requestId; this.textMessage = textMessage; this.outcome = outcome; + this.trace = trace; } /** @@ -76,6 +90,9 @@ public class Response { public long getRequestId() { return requestId; } + /** Returns the trace of this operation, or null if there is none. */ + public Trace getTrace() { return trace; } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java index 24fd47ed12c..1cee3249032 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -11,6 +11,8 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import java.time.Duration; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; + /** * A session for synchronous access to a document repository. This class * provides simple document access where throughput is not a concern. @@ -35,6 +37,18 @@ public interface SyncSession extends Session { * @param priority the priority with which to perform this operation */ default void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { + put(documentPut, parameters().withPriority(priority)); + } + + /** + * Puts a document. When this method returns, the document is safely received. + * + * @param documentPut the DocumentPut operation + * @param parameters parameters for the operation + * + * @param documentPut the DocumentPut operation + */ + default void put(DocumentPut documentPut, DocumentOperationParameters parameters) { put(documentPut); } @@ -85,6 +99,20 @@ public interface SyncSession extends Session { Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout); /** + * Gets a document with timeout. + * + * @param id the id of the document to get + * @param parameters parameters for the operation + * @param timeout timeout. If timeout is null, an unspecified default will be used + * @return the known document having this id, or null if there is no document having this id + * @throws UnsupportedOperationException thrown if this access does not support retrieving + * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT}) + */ + default Document get(DocumentId id, DocumentOperationParameters parameters, Duration timeout) { + return get(id, timeout); + } + + /** * Removes a document if it is present and condition is fulfilled. * * @param documentRemove document to delete @@ -103,6 +131,18 @@ public interface SyncSession extends Session { boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority); /** + * Removes a document if it is present. + * + * @param documentRemove document remove operation + * @param parameters parameters for the operation + * @return true if the document with this id was removed, false otherwise. + * @throws UnsupportedOperationException thrown if this access does not support removal + */ + default boolean remove(DocumentRemove documentRemove, DocumentOperationParameters parameters) { + return remove(documentRemove); + } + + /** * Updates a document. * * @param update the updates to perform @@ -127,4 +167,19 @@ public interface SyncSession extends Session { */ boolean update(DocumentUpdate update, DocumentProtocol.Priority priority); + /** + * Updates a document. + * + * @param update the updates to perform. + * @param parameters parameters for the operation + * @return false if the updates could not be applied as the document does not exist and + * {@link DocumentUpdate#setCreateIfNonExistent(boolean) create-if-non-existent} is not set. + * @throws DocumentAccessException on update error, including but not limited to: 1. timeouts, + * 2. the document exists but the {@link DocumentUpdate#setCondition(TestAndSetCondition) condition} + * is not met. + */ + default boolean update(DocumentUpdate update, DocumentOperationParameters parameters) { + return update(update); + } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java index 96bf58c1e64..54c338dc6c9 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi; +import com.yahoo.messagebus.Trace; + /** * This response is provided for successful document update operations. Use the * wasFound() method to check whether or not the document was actually found. @@ -12,7 +14,11 @@ public class UpdateResponse extends Response { private final boolean wasFound; public UpdateResponse(long requestId, boolean wasFound) { - super(requestId); + this(requestId, wasFound, null); + } + + public UpdateResponse(long requestId, boolean wasFound, Trace trace) { + super(requestId, null, wasFound ? Outcome.SUCCESS : Outcome.NOT_FOUND, trace); this.wasFound = wasFound; } @@ -21,6 +27,10 @@ public class UpdateResponse extends Response { } @Override + // TODO: fix this when/if NOT_FOUND is no longer a success. + public boolean isSuccess() { return super.isSuccess() || outcome() == Outcome.NOT_FOUND; } + + @Override public int hashCode() { return super.hashCode() + Boolean.valueOf(wasFound).hashCode(); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java index 398675e594e..40f26a82a89 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java @@ -61,7 +61,7 @@ public class LocalAsyncSession implements AsyncSession { long req = getNextRequestId(); try { syncSession.put(documentPut, pri); - addResponse(new DocumentResponse(req)); + addResponse(new DocumentResponse(req, documentPut.getDocument())); } catch (Exception e) { addResponse(new DocumentResponse(req, documentPut.getDocument(), e.getMessage(), Response.Outcome.ERROR)); } @@ -85,7 +85,7 @@ public class LocalAsyncSession implements AsyncSession { try { addResponse(new DocumentResponse(req, syncSession.get(id))); } catch (Exception e) { - addResponse(new DocumentResponse(req, e.getMessage(), Response.Outcome.ERROR)); + addResponse(new DocumentResponse(req, null, e.getMessage(), Response.Outcome.ERROR)); } return new Result(req); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 7471d285db1..7a71089c180 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -9,6 +9,7 @@ import com.yahoo.document.fieldset.AllFields; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentIdResponse; +import com.yahoo.documentapi.DocumentOperationParameters; import com.yahoo.documentapi.DocumentResponse; import com.yahoo.documentapi.DocumentUpdateResponse; import com.yahoo.documentapi.RemoveResponse; @@ -41,9 +42,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; import static com.yahoo.documentapi.Response.Outcome.CONDITION_FAILED; import static com.yahoo.documentapi.Response.Outcome.ERROR; import static com.yahoo.documentapi.Response.Outcome.NOT_FOUND; +import static com.yahoo.documentapi.Response.Outcome.SUCCESS; /** * An access session which wraps a messagebus source session sending document messages. @@ -96,19 +99,24 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result put(Document document) { - return put(new DocumentPut(document), DocumentProtocol.Priority.NORMAL_3); + return put(new DocumentPut(document), parameters()); } @Override public Result put(DocumentPut documentPut, DocumentProtocol.Priority pri) { + return put(documentPut, parameters().withPriority(pri)); + } + + @Override + public Result put(DocumentPut documentPut, DocumentOperationParameters parameters) { PutDocumentMessage msg = new PutDocumentMessage(documentPut); - msg.setPriority(pri); - return send(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_3)); + return send(msg, parameters); } @Override public Result get(DocumentId id) { - return get(id, DocumentProtocol.Priority.NORMAL_1); + return get(id, parameters()); } @Override @@ -119,35 +127,52 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public Result get(DocumentId id, DocumentProtocol.Priority pri) { - GetDocumentMessage msg = new GetDocumentMessage(id, AllFields.NAME); - msg.setPriority(pri); - return send(msg); + return get(id, parameters().withPriority(pri)); + } + + @Override + public Result get(DocumentId id, DocumentOperationParameters parameters) { + GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); + return send(msg, parameters); } @Override public Result remove(DocumentId id) { - return remove(id, DocumentProtocol.Priority.NORMAL_2); + return remove(id, parameters()); } @Override public Result remove(DocumentId id, DocumentProtocol.Priority pri) { + return remove(id, parameters().withPriority(pri)); + } + + @Override + public Result remove(DocumentId id, DocumentOperationParameters parameters) { RemoveDocumentMessage msg = new RemoveDocumentMessage(id); - msg.setPriority(pri); - return send(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); + return send(msg, parameters); } @Override public Result update(DocumentUpdate update) { - return update(update, DocumentProtocol.Priority.NORMAL_2); + return update(update, parameters()); } @Override public Result update(DocumentUpdate update, DocumentProtocol.Priority pri) { + return update(update, parameters().withPriority(pri)); + } + + @Override + public Result update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setPriority(pri); - return send(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); + return send(msg, parameters); } + // TODO jonmv: Was this done to remedy get route no longer being possible to set through doc/v1 after default-get was added? + // TODO jonmv: If so, this is no longer needed with doc/v1.1 and later. private boolean mayOverrideWithGetOnlyRoute(Message msg) { // Only allow implicitly overriding the default Get route if the message is attempted sent // with the default route originally. Otherwise it's reasonable to assume that the caller @@ -156,19 +181,13 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { && ("default".equals(route) || "route:default".equals(route))); } - /** - * A convenience method for assigning the internal trace level and route string to a message before sending it - * through the internal mbus session object. - * - * @param msg the message to send. - * @return the document api result object. - */ - public Result send(Message msg) { + Result send(Message msg, DocumentOperationParameters parameters) { try { long reqId = requestId.incrementAndGet(); msg.setContext(reqId); - msg.getTrace().setLevel(traceLevel); - String toRoute = (mayOverrideWithGetOnlyRoute(msg) ? routeForGet : route); + msg.getTrace().setLevel(parameters.traceLevel().orElse(traceLevel)); + // Use route from parameters, or session route if non-default, or finally, defaults for get and non-get, if set. Phew! + String toRoute = parameters.route().orElse(mayOverrideWithGetOnlyRoute(msg) ? routeForGet : route); if (toRoute != null) { return toResult(reqId, session.send(msg, toRoute, true)); } else { @@ -179,6 +198,17 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } } + /** + * A convenience method for assigning the internal trace level and route string to a message before sending it + * through the internal mbus session object. + * + * @param msg the message to send. + * @return the document api result object. + */ + public Result send(Message msg) { + return send(msg, null); + } + @Override public Response getNext() { return responses.poll(); @@ -256,7 +286,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } private static Response toResponse(Reply reply) { - long reqId = (Long)reply.getContext(); + long reqId = (Long) reply.getContext(); return reply.hasErrors() ? toError(reply, reqId) : toSuccess(reply, reqId); } @@ -269,15 +299,15 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { String err = getErrorMessage(reply); switch (msg.getType()) { case DocumentProtocol.MESSAGE_PUTDOCUMENT: - return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), err, outcome); + return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), err, outcome, reply.getTrace()); case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), err, outcome); + return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), err, outcome, reply.getTrace()); case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), err, outcome); + return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), err, outcome, reply.getTrace()); case DocumentProtocol.MESSAGE_GETDOCUMENT: - return new DocumentIdResponse(reqId, ((GetDocumentMessage)msg).getDocumentId(), err, outcome); + return new DocumentIdResponse(reqId, ((GetDocumentMessage)msg).getDocumentId(), err, outcome, reply.getTrace()); default: - return new Response(reqId, err, outcome); + return new Response(reqId, err, outcome, reply.getTrace()); } } @@ -289,26 +319,15 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { if (getDoc != null) { getDoc.setLastModified(docReply.getLastModified()); } - return new DocumentResponse(reqId, getDoc); + return new DocumentResponse(reqId, getDoc, reply.getTrace()); case DocumentProtocol.REPLY_REMOVEDOCUMENT: - return new RemoveResponse(reqId, ((RemoveDocumentReply)reply).wasFound()); + return new RemoveResponse(reqId, ((RemoveDocumentReply)reply).wasFound(), reply.getTrace()); case DocumentProtocol.REPLY_UPDATEDOCUMENT: - return new UpdateResponse(reqId, ((UpdateDocumentReply)reply).wasFound()); + return new UpdateResponse(reqId, ((UpdateDocumentReply)reply).wasFound(), reply.getTrace()); case DocumentProtocol.REPLY_PUTDOCUMENT: - break; - default: - return new Response(reqId); - } - Message msg = reply.getMessage(); - switch (msg.getType()) { - case DocumentProtocol.MESSAGE_PUTDOCUMENT: - return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument()); - case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId()); - case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate()); + return new DocumentResponse(reqId, ((PutDocumentMessage)reply.getMessage()).getDocumentPut().getDocument(), reply.getTrace()); default: - return new Response(reqId); + return new Response(reqId, null, SUCCESS, reply.getTrace()); } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index ff891dbb298..9832529c157 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -20,6 +20,8 @@ import com.yahoo.messagebus.network.local.LocalNetwork; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.logging.Level; +import java.util.logging.Logger; /** * This class implements the {@link DocumentAccess} interface using message bus for communication. @@ -29,6 +31,8 @@ import java.util.concurrent.ScheduledExecutorService; */ public class MessageBusDocumentAccess extends DocumentAccess { + private static final Logger log = Logger.getLogger(MessageBusDocumentAccess.class.getName()); + private final NetworkMessageBus bus; private final MessageBusParams params; @@ -60,7 +64,12 @@ public class MessageBusDocumentAccess extends DocumentAccess { bus = new NetworkMessageBus(network, new MessageBus(network, mbusParams)); } else { - bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); + if (params.getRPCNetworkParams().getSlobroksConfig() != null && mbusParams.getMessageBusConfig() != null) + bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams()); + else { + log.log(Level.FINE, () -> "Setting up self-subscription to config because explicit config was missing; try to avoid this in containers"); + bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); + } } } catch (Exception e) { @@ -92,7 +101,7 @@ public class MessageBusDocumentAccess extends DocumentAccess { @Override public MessageBusVisitorSession createVisitorSession(VisitorParameters params) throws ParseException, IllegalArgumentException { MessageBusVisitorSession session = MessageBusVisitorSession.createForMessageBus( - bus.getMessageBus(), scheduledExecutorService, params); + messageBus(), scheduledExecutorService, params); session.start(); return session; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index a9621950b88..c7ab8a23e11 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -9,6 +9,7 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.fieldset.AllFields; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.DocumentAccessException; +import com.yahoo.documentapi.DocumentOperationParameters; import com.yahoo.documentapi.Response; import com.yahoo.documentapi.Result; import com.yahoo.documentapi.SyncParameters; @@ -28,6 +29,8 @@ import com.yahoo.messagebus.ReplyHandler; import java.time.Duration; +import static com.yahoo.documentapi.DocumentOperationParameters.parameters; + /** * An implementation of the SyncSession interface running over message bus. * @@ -84,10 +87,14 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re * @return The reply received. */ public Reply syncSend(Message msg) { - return syncSend(msg, defaultTimeout); + return syncSend(msg, parameters()); + } + + private Reply syncSend(Message msg, DocumentOperationParameters parameters) { + return syncSend(msg, defaultTimeout, parameters()); } - private Reply syncSend(Message msg, Duration timeout) { + private Reply syncSend(Message msg, Duration timeout, DocumentOperationParameters parameters) { if (timeout != null) { msg.setTimeRemaining(timeout.toMillis()); } @@ -97,7 +104,7 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re msg.pushHandler(this); // store monitor Result result = null; while (result == null || result.type() == Result.ResultType.TRANSIENT_ERROR) { - result = session.send(msg); + result = session.send(msg, parameters); if (result != null && result.isSuccess()) { break; } @@ -114,37 +121,40 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public void put(DocumentPut documentPut) { - put(documentPut, DocumentProtocol.Priority.NORMAL_3); + put(documentPut, parameters()); } @Override public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { - PutDocumentMessage msg = new PutDocumentMessage(documentPut); - msg.setPriority(priority); - syncSendPutDocumentMessage(msg); + put(documentPut, parameters().withPriority(priority)); } @Override - public Document get(DocumentId id) { - return get(id, null); + public void put(DocumentPut documentPut, DocumentOperationParameters parameters) { + PutDocumentMessage msg = new PutDocumentMessage(documentPut); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_3)); + Reply reply = syncSend(msg, parameters); + if (reply.hasErrors()) { + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); + } } @Override - public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) { - return get(id, fieldSet, pri, null); + public Document get(DocumentId id, Duration timeout) { + return get(id, parameters(), timeout); } @Override - public Document get(DocumentId id, Duration timeout) { - return get(id, AllFields.NAME, DocumentProtocol.Priority.NORMAL_1, timeout); + public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri, Duration timeout) { + return get(id, parameters().withFieldSet(fieldSet).withPriority(pri), timeout); } @Override - public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri, Duration timeout) { - GetDocumentMessage msg = new GetDocumentMessage(id, fieldSet); - msg.setPriority(pri); + public Document get(DocumentId id, DocumentOperationParameters parameters, Duration timeout) { + GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); - Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout); + Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply)); } @@ -161,21 +171,20 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public boolean remove(DocumentRemove documentRemove) { - RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); - msg.setCondition(documentRemove.getCondition()); - return remove(msg); + return remove(documentRemove, parameters()); } @Override public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority pri) { - RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); - msg.setPriority(pri); - msg.setCondition(documentRemove.getCondition()); - return remove(msg); + return remove(documentRemove, parameters().withPriority(pri)); } - private boolean remove(RemoveDocumentMessage msg) { - Reply reply = syncSend(msg); + @Override + public boolean remove(DocumentRemove documentRemove, DocumentOperationParameters parameters) { + RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); + msg.setCondition(documentRemove.getCondition()); + Reply reply = syncSend(msg, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply)); } @@ -187,14 +196,19 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re @Override public boolean update(DocumentUpdate update) { - return update(update, DocumentProtocol.Priority.NORMAL_2); + return update(update, parameters()); } @Override public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) { + return update(update, parameters().withPriority(pri)); + } + + @Override + public boolean update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setPriority(pri); - Reply reply = syncSend(msg); + msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); + Reply reply = syncSend(msg, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); } @@ -243,10 +257,4 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re } } - private void syncSendPutDocumentMessage(PutDocumentMessage putDocumentMessage) { - Reply reply = syncSend(putDocumentMessage); - if (reply.hasErrors()) { - throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); - } - } } diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 76c1a63b881..33bd098975a 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -29,8 +29,11 @@ vespa_define_module( src/tests/eval/tensor_lambda src/tests/eval/tensor_spec src/tests/eval/value_cache + src/tests/eval/value_codec src/tests/eval/value_type src/tests/gp/ponder_nov2017 + src/tests/instruction/generic_join + src/tests/instruction/generic_rename src/tests/tensor/dense_add_dimension_optimizer src/tests/tensor/dense_dimension_combiner src/tests/tensor/dense_dot_product_function @@ -53,7 +56,9 @@ vespa_define_module( src/tests/tensor/direct_dense_tensor_builder src/tests/tensor/direct_sparse_tensor_builder src/tests/tensor/index_lookup_table + src/tests/tensor/instruction_benchmark src/tests/tensor/onnx_wrapper + src/tests/tensor/packed_mappings src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance @@ -70,8 +75,10 @@ vespa_define_module( src/vespa/eval/eval/test src/vespa/eval/eval/value_cache src/vespa/eval/gp + src/vespa/eval/instruction src/vespa/eval/tensor src/vespa/eval/tensor/dense + src/vespa/eval/tensor/mixed src/vespa/eval/tensor/serialization src/vespa/eval/tensor/sparse ) diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp index 32a099afce3..4827fa3be3c 100644 --- a/eval/src/tests/eval/simple_value/simple_value_test.cpp +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -1,12 +1,16 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/instruction/generic_join.h> +#include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> using namespace vespalib; using namespace vespalib::eval; +using namespace vespalib::eval::instruction; using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; @@ -62,24 +66,27 @@ TensorSpec simple_tensor_join(const TensorSpec &a, const TensorSpec &b, join_fun } TensorSpec simple_value_new_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { - auto lhs = new_value_from_spec(a, SimpleValueBuilderFactory()); - auto rhs = new_value_from_spec(b, SimpleValueBuilderFactory()); - auto result = new_join(*lhs, *rhs, function, SimpleValueBuilderFactory()); - return spec_from_new_value(*result); + Stash stash; + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs,*rhs}))); } TEST(SimpleValueTest, simple_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { TensorSpec expect = spec(layout, N()); - std::unique_ptr<NewValue> value = new_value_from_spec(expect, SimpleValueBuilderFactory()); - TensorSpec actual = spec_from_new_value(*value); + std::unique_ptr<Value> value = value_from_spec(expect, SimpleValueBuilderFactory::get()); + TensorSpec actual = spec_from_value(*value); EXPECT_EQ(actual, expect); } } TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { ValueType type = ValueType::from_spec("tensor<float>(x{},y[2],z{})"); - SimpleValueBuilderFactory factory; + const auto &factory = SimpleValueBuilderFactory::get(); std::unique_ptr<ValueBuilder<float>> builder = factory.create_value_builder<float>(type); float seq = 0.0; for (vespalib::string x: {"a", "b", "c"}) { @@ -92,7 +99,7 @@ TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { } seq += 100.0; } - std::unique_ptr<NewValue> value = builder->build(std::move(builder)); + std::unique_ptr<Value> value = builder->build(std::move(builder)); EXPECT_EQ(value->index().size(), 6); auto view = value->index().create_view({0}); vespalib::stringref query = "b"; @@ -108,48 +115,6 @@ TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { EXPECT_FALSE(view->next_result({&label}, subspace)); } -TEST(SimpleValueTest, dense_join_plan_can_be_created) { - auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); - auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); - auto plan = DenseJoinPlan(lhs, rhs); - std::vector<size_t> expect_loop = {30,4,6}; - std::vector<size_t> expect_lhs_stride = {6,0,1}; - std::vector<size_t> expect_rhs_stride = {4,1,0}; - EXPECT_EQ(plan.lhs_size, 180); - EXPECT_EQ(plan.rhs_size, 120); - EXPECT_EQ(plan.out_size, 720); - EXPECT_EQ(plan.loop_cnt, expect_loop); - EXPECT_EQ(plan.lhs_stride, expect_lhs_stride); - EXPECT_EQ(plan.rhs_stride, expect_rhs_stride); -} - -TEST(SimpleValueTest, sparse_join_plan_can_be_created) { - auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); - auto rhs = ValueType::from_spec("tensor(b[6],c[5],d[4],g{},h{})"); - auto plan = SparseJoinPlan(lhs, rhs); - using SRC = SparseJoinPlan::Source; - std::vector<SRC> expect_sources = {SRC::LHS,SRC::BOTH,SRC::RHS}; - std::vector<size_t> expect_lhs_overlap = {1}; - std::vector<size_t> expect_rhs_overlap = {0}; - EXPECT_EQ(plan.sources, expect_sources); - EXPECT_EQ(plan.lhs_overlap, expect_lhs_overlap); - EXPECT_EQ(plan.rhs_overlap, expect_rhs_overlap); -} - -TEST(SimpleValueTest, dense_join_plan_can_be_executed) { - auto plan = DenseJoinPlan(ValueType::from_spec("tensor(a[2])"), - ValueType::from_spec("tensor(b[3])")); - std::vector<int> a({1, 2}); - std::vector<int> b({3, 4, 5}); - std::vector<int> c(6, 0); - std::vector<int> expect = {3,4,5,6,8,10}; - ASSERT_EQ(plan.out_size, 6); - int *dst = &c[0]; - auto cell_join = [&](size_t a_idx, size_t b_idx) { *dst++ = (a[a_idx] * b[b_idx]); }; - plan.execute(0, 0, cell_join); - EXPECT_EQ(c, expect); -} - TEST(SimpleValueTest, new_generic_join_works_for_simple_values) { ASSERT_TRUE((join_layouts.size() % 2) == 0); for (size_t i = 0; i < join_layouts.size(); i += 2) { diff --git a/eval/src/tests/eval/value_codec/CMakeLists.txt b/eval/src/tests/eval/value_codec/CMakeLists.txt new file mode 100644 index 00000000000..aa1adf10136 --- /dev/null +++ b/eval/src/tests/eval/value_codec/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(eval_value_codec_test_app TEST + SOURCES + value_codec_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_value_codec_test_app COMMAND eval_value_codec_test_app) diff --git a/eval/src/tests/eval/value_codec/value_codec_test.cpp b/eval/src/tests/eval/value_codec/value_codec_test.cpp new file mode 100644 index 00000000000..00f37e1f87a --- /dev/null +++ b/eval/src/tests/eval/value_codec/value_codec_test.cpp @@ -0,0 +1,271 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <iostream> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/vespalib/data/memory.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/objects/nbostream.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +const ValueBuilderFactory &factory = SimpleValueBuilderFactory::get(); + +std::vector<Layout> layouts = { + {}, + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + + +TEST(ValueCodecTest, simple_values_can_be_converted_from_and_to_tensor_spec) { + for (const auto &layout: layouts) { + TensorSpec expect = spec(layout, N()); + std::unique_ptr<Value> value = value_from_spec(expect, factory); + TensorSpec actual = spec_from_value(*value); + EXPECT_EQ(actual, expect); + } +} + +TEST(ValueCodecTest, simple_values_can_be_built_using_tensor_spec) { + TensorSpec spec("tensor(w{},x[2],y{},z[2])"); + spec.add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); + Value::UP tensor = value_from_spec(spec, factory); + TensorSpec full_spec("tensor(w{},x[2],y{},z[2])"); + full_spec + .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 1.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "xxx"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 2.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 0.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "xxx"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "xxx"}, {"z", 0}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "yyy"}, {"x", 0}, {"y", "yyy"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 0}}, 3.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "xxx"}, {"z", 1}}, 0.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) + .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); + Value::UP full_tensor = value_from_spec(full_spec, factory); + EXPECT_EQUAL(full_spec, spec_from_value(*tensor)); + EXPECT_EQUAL(full_spec, spec_from_value(*full_tensor)); +}; + +//----------------------------------------------------------------------------- + +vespalib::string make_type_spec(bool use_float, const vespalib::string &dims) { + vespalib::string type_spec = "tensor"; + if (use_float) { + type_spec.append("<float>"); + } + type_spec.append(dims); + return type_spec; +} + +struct TensorExample { + virtual ~TensorExample(); + virtual TensorSpec make_spec(bool use_float) const = 0; + virtual std::unique_ptr<Value> make_tensor(bool use_float) const = 0; + virtual void encode_default(nbostream &dst) const = 0; + virtual void encode_with_double(nbostream &dst) const = 0; + virtual void encode_with_float(nbostream &dst) const = 0; + void verify_encode_decode() const { + nbostream expect_default; + nbostream expect_double; + nbostream expect_float; + encode_default(expect_default); + encode_with_double(expect_double); + encode_with_float(expect_float); + nbostream data_double; + nbostream data_float; + encode_value(*make_tensor(false), data_double); + encode_value(*make_tensor(true), data_float); + EXPECT_EQ(Memory(data_double.peek(), data_double.size()), + Memory(expect_default.peek(), expect_default.size())); + EXPECT_EQ(Memory(data_float.peek(), data_float.size()), + Memory(expect_float.peek(), expect_float.size())); + EXPECT_EQ(spec_from_value(*decode_value(expect_default, factory)), make_spec(false)); + EXPECT_EQ(spec_from_value(*decode_value(expect_double, factory)), make_spec(false)); + EXPECT_EQ(spec_from_value(*decode_value(expect_float, factory)), make_spec(true)); + } +}; +TensorExample::~TensorExample() = default; + +//----------------------------------------------------------------------------- + +struct SparseTensorExample : TensorExample { + TensorSpec make_spec(bool use_float) const override { + return TensorSpec(make_type_spec(use_float, "(x{},y{})")) + .add({{"x","a"},{"y","a"}}, 1) + .add({{"x","a"},{"y","b"}}, 2) + .add({{"x","b"},{"y","a"}}, 3); + } + std::unique_ptr<Value> make_tensor(bool use_float) const override { + return value_from_spec(make_spec(use_float), factory); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 2; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 3; + } + void encode_default(nbostream &dst) const override { + dst.putInt1_4Bytes(1); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const override { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const override { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST(ValueCodecTest, sparse_tensors_can_be_encoded_and_decoded) { + SparseTensorExample f1; + f1.verify_encode_decode(); +} + +//----------------------------------------------------------------------------- + +struct DenseTensorExample : TensorExample { + TensorSpec make_spec(bool use_float) const override { + return TensorSpec(make_type_spec(use_float, "(x[3],y[2])")) + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 2) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 4) + .add({{"x",2},{"y",0}}, 5) + .add({{"x",2},{"y",1}}, 6); + } + std::unique_ptr<Value> make_tensor(bool use_float) const override { + return value_from_spec(make_spec(use_float), factory); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(2); + dst << (T) 1; + dst << (T) 2; + dst << (T) 3; + dst << (T) 4; + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const override { + dst.putInt1_4Bytes(2); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const override { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const override { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST(ValueCodecTest, dense_tensors_can_be_encoded_and_decoded) { + DenseTensorExample f1; + f1.verify_encode_decode(); +} + +//----------------------------------------------------------------------------- + +struct MixedTensorExample : TensorExample { + TensorSpec make_spec(bool use_float) const override { + return TensorSpec(make_type_spec(use_float, "(x{},y{},z[2])")) + .add({{"x","a"},{"y","a"},{"z",0}}, 1) + .add({{"x","a"},{"y","a"},{"z",1}}, 2) + .add({{"x","a"},{"y","b"},{"z",0}}, 3) + .add({{"x","a"},{"y","b"},{"z",1}}, 4) + .add({{"x","b"},{"y","a"},{"z",0}}, 5) + .add({{"x","b"},{"y","a"},{"z",1}}, 6); + } + std::unique_ptr<Value> make_tensor(bool use_float) const override { + return value_from_spec(make_spec(use_float), factory); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(1); + dst.writeSmallString("z"); + dst.putInt1_4Bytes(2); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst << (T) 2; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 3; + dst << (T) 4; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const override { + dst.putInt1_4Bytes(3); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const override { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const override { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST(ValueCodecTest, mixed_tensors_can_be_encoded_and_decoded) { + MixedTensorExample f1; + f1.verify_encode_decode(); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 2b103a91b81..9f1519ee7c0 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -325,12 +325,17 @@ TEST("require that type-related predicate functions work as expected") { TEST_DO(verify_predicates(type("tensor<float>(x[5],y{})"), false, false, true, false, false)); } -TEST("require that mapped dimensions can be counted") { +TEST("require that mapped and indexed dimensions can be counted") { EXPECT_EQUAL(type("double").count_mapped_dimensions(), 0u); + EXPECT_EQUAL(type("double").count_indexed_dimensions(), 0u); EXPECT_EQUAL(type("tensor(x[5],y[5])").count_mapped_dimensions(), 0u); + EXPECT_EQUAL(type("tensor(x[5],y[5])").count_indexed_dimensions(), 2u); EXPECT_EQUAL(type("tensor(x{},y[5])").count_mapped_dimensions(), 1u); - EXPECT_EQUAL(type("tensor(x[5],y{})").count_mapped_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x{},y[5])").count_indexed_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x[1],y{})").count_mapped_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x[1],y{})").count_indexed_dimensions(), 1u); EXPECT_EQUAL(type("tensor(x{},y{})").count_mapped_dimensions(), 2u); + EXPECT_EQUAL(type("tensor(x{},y{})").count_indexed_dimensions(), 0u); } TEST("require that dense subspace size calculation works as expected") { diff --git a/eval/src/tests/instruction/generic_join/CMakeLists.txt b/eval/src/tests/instruction/generic_join/CMakeLists.txt new file mode 100644 index 00000000000..13fc6550d3c --- /dev/null +++ b/eval/src/tests/instruction/generic_join/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_generic_join_test_app TEST + SOURCES + generic_join_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_generic_join_test_app COMMAND eval_generic_join_test_app) diff --git a/eval/src/tests/instruction/generic_join/generic_join_test.cpp b/eval/src/tests/instruction/generic_join/generic_join_test.cpp new file mode 100644 index 00000000000..4821bf092da --- /dev/null +++ b/eval/src/tests/instruction/generic_join/generic_join_test.cpp @@ -0,0 +1,138 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/instruction/generic_join.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::instruction; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector<Layout> join_layouts = { + {}, {}, + {x(5)}, {x(5)}, + {x(5)}, {y(5)}, + {x(5)}, {x(5),y(5)}, + {y(3)}, {x(2),z(3)}, + {x(3),y(5)}, {y(5),z(7)}, + float_cells({x(3),y(5)}), {y(5),z(7)}, + {x(3),y(5)}, float_cells({y(5),z(7)}), + float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {y({"foo","bar","baz"})}, + {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, + {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), + float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) +}; + +bool join_address(const TensorSpec::Address &a, const TensorSpec::Address &b, TensorSpec::Address &addr) { + for (const auto &dim_a: a) { + auto pos_b = b.find(dim_a.first); + if ((pos_b != b.end()) && !(pos_b->second == dim_a.second)) { + return false; + } + addr.insert_or_assign(dim_a.first, dim_a.second); + } + return true; +} + +TensorSpec reference_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + ValueType res_type = ValueType::join(ValueType::from_spec(a.type()), ValueType::from_spec(b.type())); + EXPECT_FALSE(res_type.is_error()); + TensorSpec result(res_type.to_spec()); + for (const auto &cell_a: a.cells()) { + for (const auto &cell_b: b.cells()) { + TensorSpec::Address addr; + if (join_address(cell_a.first, cell_b.first, addr) && + join_address(cell_b.first, cell_a.first, addr)) + { + result.add(addr, function(cell_a.second, cell_b.second)); + } + } + } + return result; +} + +TensorSpec perform_generic_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &factory = SimpleValueBuilderFactory::get(); + auto lhs = value_from_spec(a, factory); + auto rhs = value_from_spec(b, factory); + auto my_op = GenericJoin::make_instruction(lhs->type(), rhs->type(), function, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs,*rhs}))); +} + +TEST(GenericJoinTest, dense_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); + auto plan = DenseJoinPlan(lhs, rhs); + std::vector<size_t> expect_loop = {30,4,6}; + std::vector<size_t> expect_lhs_stride = {6,0,1}; + std::vector<size_t> expect_rhs_stride = {4,1,0}; + EXPECT_EQ(plan.lhs_size, 180); + EXPECT_EQ(plan.rhs_size, 120); + EXPECT_EQ(plan.out_size, 720); + EXPECT_EQ(plan.loop_cnt, expect_loop); + EXPECT_EQ(plan.lhs_stride, expect_lhs_stride); + EXPECT_EQ(plan.rhs_stride, expect_rhs_stride); +} + +TEST(GenericJoinTest, sparse_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(b[6],c[5],d[4],g{},h{})"); + auto plan = SparseJoinPlan(lhs, rhs); + using SRC = SparseJoinPlan::Source; + std::vector<SRC> expect_sources = {SRC::LHS,SRC::BOTH,SRC::RHS}; + std::vector<size_t> expect_lhs_overlap = {1}; + std::vector<size_t> expect_rhs_overlap = {0}; + EXPECT_EQ(plan.sources, expect_sources); + EXPECT_EQ(plan.lhs_overlap, expect_lhs_overlap); + EXPECT_EQ(plan.rhs_overlap, expect_rhs_overlap); +} + +TEST(GenericJoinTest, dense_join_plan_can_be_executed) { + auto plan = DenseJoinPlan(ValueType::from_spec("tensor(a[2])"), + ValueType::from_spec("tensor(b[3])")); + std::vector<int> a({1, 2}); + std::vector<int> b({3, 4, 5}); + std::vector<int> c(6, 0); + std::vector<int> expect = {3,4,5,6,8,10}; + ASSERT_EQ(plan.out_size, 6); + int *dst = &c[0]; + auto cell_join = [&](size_t a_idx, size_t b_idx) { *dst++ = (a[a_idx] * b[b_idx]); }; + plan.execute(0, 0, cell_join); + EXPECT_EQ(c, expect); +} + +TEST(GenericJoinTest, generic_join_works_for_simple_values) { + ASSERT_TRUE((join_layouts.size() % 2) == 0); + for (size_t i = 0; i < join_layouts.size(); i += 2) { + TensorSpec lhs = spec(join_layouts[i], Div16(N())); + TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = reference_join(lhs, rhs, fun); + auto actual = perform_generic_join(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/generic_rename/CMakeLists.txt b/eval/src/tests/instruction/generic_rename/CMakeLists.txt new file mode 100644 index 00000000000..98af0fe0212 --- /dev/null +++ b/eval/src/tests/instruction/generic_rename/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_generic_rename_test_app TEST + SOURCES + generic_rename_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_generic_rename_test_app COMMAND eval_generic_rename_test_app) diff --git a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp new file mode 100644 index 00000000000..f61899e4dda --- /dev/null +++ b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp @@ -0,0 +1,145 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/instruction/generic_rename.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::instruction; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector<Layout> rename_layouts = { + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + +struct FromTo { + std::vector<vespalib::string> from; + std::vector<vespalib::string> to; +}; + +std::vector<FromTo> rename_from_to = { + { {"x"}, {"x_renamed"} }, + { {"x"}, {"z_was_x"} }, + { {"x", "y"}, {"y", "x"} }, + { {"x", "z"}, {"z", "x"} }, + { {"x", "y", "z"}, {"a", "b", "c"} }, + { {"z"}, {"a"} }, + { {"y"}, {"z_was_y"} }, + { {"y"}, {"b"} } +}; + + +TEST(GenericRenameTest, dense_rename_plan_can_be_created_and_executed) { + auto lhs = ValueType::from_spec("tensor(a[2],c[3],d{},e[5],g[7],h{})"); + std::vector<vespalib::string> from({"a", "c", "e"}); + std::vector<vespalib::string> to({"f", "a", "b"}); + ValueType renamed = lhs.rename(from, to); + auto plan = DenseRenamePlan(lhs, renamed, from, to); + std::vector<size_t> expect_loop = {15,2,7}; + std::vector<size_t> expect_stride = {7,105,1}; + EXPECT_EQ(plan.subspace_size, 210); + EXPECT_EQ(plan.loop_cnt, expect_loop); + EXPECT_EQ(plan.stride, expect_stride); + std::vector<int> out; + int want[3][5][2][7]; + size_t counter = 0; + for (size_t a = 0; a < 2; ++a) { + for (size_t c = 0; c < 3; ++c) { + for (size_t e = 0; e < 5; ++e) { + for (size_t g = 0; g < 7; ++g) { + want[c][e][a][g] = counter++; + } + } + } + } + std::vector<int> expect(210); + memcpy(&expect[0], &want[0], 210*sizeof(int)); + auto move_cell = [&](size_t offset) { out.push_back(offset); }; + plan.execute(0, move_cell); + EXPECT_EQ(out, expect); +} + +TEST(GenericRenameTest, sparse_rename_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},c{},d[3],e{},g{},h[5])"); + std::vector<vespalib::string> from({"a", "c", "e"}); + std::vector<vespalib::string> to({"f", "a", "b"}); + ValueType renamed = lhs.rename(from, to); + auto plan = SparseRenamePlan(lhs, renamed, from, to); + EXPECT_EQ(plan.mapped_dims, 4); + std::vector<size_t> expect = {2,0,1,3}; + EXPECT_EQ(plan.output_dimensions, expect); +} + +vespalib::string rename_dimension(const vespalib::string &name, const FromTo &ft) { + assert(ft.from.size() == ft.to.size()); + for (size_t i = 0; i < ft.from.size(); ++i) { + if (name == ft.from[i]) { + return ft.to[i]; + } + } + return name; +} + +TensorSpec reference_rename(const TensorSpec &a, const FromTo &ft) { + ValueType res_type = ValueType::from_spec(a.type()).rename(ft.from, ft.to); + EXPECT_FALSE(res_type.is_error()); + TensorSpec result(res_type.to_spec()); + for (const auto &cell: a.cells()) { + TensorSpec::Address addr; + for (const auto &dim: cell.first) { + addr.insert_or_assign(rename_dimension(dim.first, ft), dim.second); + } + result.add(addr, cell.second); + } + return result; +} + +TensorSpec perform_generic_rename(const TensorSpec &a, const ValueType &res_type, + const FromTo &ft, const ValueBuilderFactory &factory) +{ + Stash stash; + auto lhs = value_from_spec(a, factory); + auto my_op = GenericRename::make_instruction(lhs->type(), res_type, ft.from, ft.to, factory, stash); + InterpretedFunction::EvalSingle single(my_op); + return spec_from_value(single.eval(std::vector<Value::CREF>({*lhs}))); +} + +void test_generic_rename(const ValueBuilderFactory &factory) { + for (const auto & layout : rename_layouts) { + TensorSpec lhs = spec(layout, N()); + ValueType lhs_type = ValueType::from_spec(lhs.type()); + // printf("lhs_type: %s\n", lhs_type.to_spec().c_str()); + for (const auto & from_to : rename_from_to) { + ValueType renamed_type = lhs_type.rename(from_to.from, from_to.to); + if (renamed_type.is_error()) continue; + // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); + SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); + auto expect = reference_rename(lhs, from_to); + auto actual = perform_generic_rename(lhs, renamed_type, from_to, factory); + EXPECT_EQ(actual, expect); + } + } +} + +TEST(GenericRenameTest, generic_rename_works_for_simple_values) { + test_generic_rename(SimpleValueBuilderFactory::get()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp index 651451d81f1..e4640cf2c6a 100644 --- a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp @@ -54,7 +54,7 @@ TEST("require that tensor can be constructed") Tensor::UP tensor = buildTensor(); const SparseTensor &sparseTensor = dynamic_cast<const SparseTensor &>(*tensor); const ValueType &type = sparseTensor.type(); - const SparseTensor::Cells &cells = sparseTensor.cells(); + const SparseTensor::Cells &cells = sparseTensor.my_cells(); EXPECT_EQUAL(2u, cells.size()); assertCellValue(10, TensorAddress({{"a","1"},{"b","2"}}), type, cells); assertCellValue(20, TensorAddress({{"c","3"},{"d","4"}}), type, cells); diff --git a/eval/src/tests/tensor/instruction_benchmark/.gitignore b/eval/src/tests/tensor/instruction_benchmark/.gitignore new file mode 100644 index 00000000000..31b087883e0 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/.gitignore @@ -0,0 +1 @@ +/eval_instruction_benchmark_app diff --git a/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt b/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt new file mode 100644 index 00000000000..d2384eaf129 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_instruction_benchmark_app TEST + SOURCES + instruction_benchmark.cpp + DEPENDS + vespaeval + GTest::GTest +) diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp new file mode 100644 index 00000000000..9db7bbae4e8 --- /dev/null +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -0,0 +1,321 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +// Microbenchmark exploring performance differences between +// interpreted function instructions. + +// This benchmark was initially written to measure the difference in +// performance between (old) instructions using the TensorEngine +// immediate API and (new) instructions using the Value API +// directly. Note that all previous optimizations for dense tensors +// are trivially transformed to use the Value API, and thus only the +// generic cases need to be compared. Specifically; we want to make +// sure join performance for sparse tensors with full dimensional +// overlap does not suffer too much. Also, we want to showcase an +// improvement in generic dense join and possibly also in sparse join +// with partial dimensional overlap. Benchmarks are done using float +// cells since this is what gives best overall performance in +// production. Also, we use the multiply operation since it is the +// most optimized operations across all implementations. When +// benchmarking different implementations against each other, a smoke +// test is performed by verifying that all implementations produce the +// same result. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/instruction/generic_join.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> +#include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <optional> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::tensor; +using namespace vespalib::eval::instruction; +using vespalib::make_string_short::fmt; + +using Instruction = InterpretedFunction::Instruction; +using EvalSingle = InterpretedFunction::EvalSingle; + +template <typename T> using CREF = std::reference_wrapper<const T>; + +//----------------------------------------------------------------------------- + +struct Impl { + virtual const vespalib::string &name() const = 0; + virtual Value::UP create_value(const TensorSpec &spec) const = 0; + virtual TensorSpec create_spec(const Value &value) const = 0; + virtual Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const = 0; + virtual const TensorEngine &engine() const { return SimpleTensorEngine::ref(); } // engine used by EvalSingle + virtual ~Impl() {} +}; + +struct ValueImpl : Impl { + vespalib::string my_name; + const ValueBuilderFactory &my_factory; + ValueImpl(const vespalib::string &name_in, const ValueBuilderFactory &factory) + : my_name(name_in), my_factory(factory) {} + const vespalib::string &name() const override { return my_name; } + Value::UP create_value(const TensorSpec &spec) const override { return value_from_spec(spec, my_factory); } + TensorSpec create_spec(const Value &value) const override { return spec_from_value(value); } + Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { + return GenericJoin::make_instruction(lhs, rhs, function, my_factory, stash); + } +}; + +struct EngineImpl : Impl { + vespalib::string my_name; + const TensorEngine &my_engine; + EngineImpl(const vespalib::string &name_in, const TensorEngine &engine_in) + : my_name(name_in), my_engine(engine_in) {} + const vespalib::string &name() const override { return my_name; } + Value::UP create_value(const TensorSpec &spec) const override { return my_engine.from_spec(spec); } + TensorSpec create_spec(const Value &value) const override { return my_engine.to_spec(value); } + Instruction create_join(const ValueType &lhs, const ValueType &rhs, operation::op2_t function, Stash &stash) const override { + // create a complete tensor function joining two parameters, but only compile the join instruction itself + const auto &lhs_node = tensor_function::inject(lhs, 0, stash); + const auto &rhs_node = tensor_function::inject(rhs, 1, stash); + const auto &join_node = tensor_function::join(lhs_node, rhs_node, function, stash); + return join_node.compile_self(my_engine, stash); + } + const TensorEngine &engine() const override { return my_engine; } +}; + +//----------------------------------------------------------------------------- + +EngineImpl simple_tensor_engine_impl(" [SimpleTensorEngine]", SimpleTensorEngine::ref()); +EngineImpl default_tensor_engine_impl("[DefaultTensorEngine]", DefaultTensorEngine::ref()); +ValueImpl simple_value_impl(" [SimpleValue]", SimpleValueBuilderFactory::get()); +ValueImpl packed_mixed_tensor_impl(" [PackedMixedTensor]", PackedMixedTensorBuilderFactory::get()); + +double budget = 5.0; +std::vector<CREF<Impl>> impl_list = {simple_tensor_engine_impl, + default_tensor_engine_impl, + simple_value_impl, + packed_mixed_tensor_impl}; + +//----------------------------------------------------------------------------- + +struct EvalOp { + using UP = std::unique_ptr<EvalOp>; + const Impl &impl; + std::vector<Value::UP> values; + std::vector<Value::CREF> stack; + EvalSingle single; + EvalOp(const EvalOp &) = delete; + EvalOp &operator=(const EvalOp &) = delete; + EvalOp(Instruction op, const std::vector<CREF<TensorSpec>> &stack_spec, const Impl &impl_in) + : impl(impl_in), values(), stack(), single(impl.engine(), op) + { + for (const TensorSpec &spec: stack_spec) { + values.push_back(impl.create_value(spec)); + } + for (const auto &value: values) { + stack.push_back(*value.get()); + } + } + TensorSpec result() { return impl.create_spec(single.eval(stack)); } + double estimate_cost_us() { + auto actual = [&](){ single.eval(stack); }; + return BenchmarkTimer::benchmark(actual, budget) * 1000.0 * 1000.0; + } +}; + +//----------------------------------------------------------------------------- + +void benchmark(const vespalib::string &desc, const std::vector<EvalOp::UP> &list) { + fprintf(stderr, "--------------------------------------------------------\n"); + fprintf(stderr, "Benchmark Case: [%s]\n", desc.c_str()); + std::optional<TensorSpec> expect = std::nullopt; + for (const auto &eval: list) { + if (expect.has_value()) { + ASSERT_EQ(eval->result(), expect.value()); + } else { + expect = eval->result(); + } + } + for (const auto &eval: list) { + fprintf(stderr, " %s: %10.3f us\n", eval->impl.name().c_str(), eval->estimate_cost_us()); + } + fprintf(stderr, "--------------------------------------------------------\n"); +} + +//----------------------------------------------------------------------------- + +void benchmark_join(const vespalib::string &desc, const TensorSpec &lhs, + const TensorSpec &rhs, operation::op2_t function) +{ + Stash stash; + ValueType lhs_type = ValueType::from_spec(lhs.type()); + ValueType rhs_type = ValueType::from_spec(rhs.type()); + ValueType res_type = ValueType::join(lhs_type, rhs_type); + ASSERT_FALSE(lhs_type.is_error()); + ASSERT_FALSE(rhs_type.is_error()); + ASSERT_FALSE(res_type.is_error()); + std::vector<EvalOp::UP> list; + for (const Impl &impl: impl_list) { + auto op = impl.create_join(lhs_type, rhs_type, function, stash); + std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); + list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + } + benchmark(desc, list); +} + +//----------------------------------------------------------------------------- + +struct D { + vespalib::string name; + bool mapped; + size_t size; + size_t stride; + static D map(const vespalib::string &name_in, size_t size_in, size_t stride_in) { return D{name_in, true, size_in, stride_in}; } + static D idx(const vespalib::string &name_in, size_t size_in) { return D{name_in, false, size_in, 1}; } + operator ValueType::Dimension() const { + if (mapped) { + return ValueType::Dimension(name); + } else { + return ValueType::Dimension(name, size); + } + } + TensorSpec::Label operator()(size_t idx) const { + if (mapped) { + return TensorSpec::Label(fmt("label_%zu", idx)); + } else { + return TensorSpec::Label(idx); + } + } +}; + +void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr) { + spec.add(addr, seq); + seq += 1.0; +} + +template <typename ...Ds> void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr, const D &d, const Ds &...ds) { + for (size_t i = 0, idx = 0; i < d.size; ++i, idx += d.stride) { + addr.insert_or_assign(d.name, d(idx)); + add_cells(spec, seq, addr, ds...); + } +} + +template <typename ...Ds> TensorSpec make_spec(double seq, const Ds &...ds) { + TensorSpec spec(ValueType::tensor_type({ds...}, ValueType::CellType::FLOAT).to_spec()); + add_cells(spec, seq, TensorSpec::Address(), ds...); + return spec; +} + +TensorSpec make_vector(const D &d1, double seq) { return make_spec(seq, d1); } +TensorSpec make_cube(const D &d1, const D &d2, const D &d3, double seq) { return make_spec(seq, d1, d2, d3); } + +//----------------------------------------------------------------------------- + +TEST(MakeInputTest, print_some_test_input) { + auto number = make_spec(5.0); + auto sparse = make_vector(D::map("x", 5, 3), 1.0); + auto dense = make_vector(D::idx("x", 5), 10.0); + auto mixed = make_cube(D::map("x", 3, 7), D::idx("y", 2), D::idx("z", 2), 100.0); + fprintf(stderr, "--------------------------------------------------------\n"); + fprintf(stderr, "simple number: %s\n", number.to_string().c_str()); + fprintf(stderr, "sparse vector: %s\n", sparse.to_string().c_str()); + fprintf(stderr, "dense vector: %s\n", dense.to_string().c_str()); + fprintf(stderr, "mixed cube: %s\n", mixed.to_string().c_str()); + fprintf(stderr, "--------------------------------------------------------\n"); +} + +//----------------------------------------------------------------------------- + +TEST(NumberJoin, plain_op2) { + auto lhs = make_spec(2.0); + auto rhs = make_spec(3.0); + benchmark_join("simple numbers multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(DenseJoin, small_vectors) { + auto lhs = make_vector(D::idx("x", 10), 1.0); + auto rhs = make_vector(D::idx("x", 10), 2.0); + benchmark_join("small dense vector multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, full_overlap) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); + auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 2.0); + benchmark_join("dense full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, partial_overlap) { + auto lhs = make_cube(D::idx("a", 8), D::idx("c", 8), D::idx("d", 8), 1.0); + auto rhs = make_cube(D::idx("b", 8), D::idx("c", 8), D::idx("d", 8), 2.0); + benchmark_join("dense partial overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(DenseJoin, no_overlap) { + auto lhs = make_cube(D::idx("a", 4), D::idx("e", 4), D::idx("f", 4), 1.0); + auto rhs = make_cube(D::idx("b", 4), D::idx("c", 4), D::idx("d", 4), 2.0); + benchmark_join("dense no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(SparseJoin, small_vectors) { + auto lhs = make_vector(D::map("x", 10, 1), 1.0); + auto rhs = make_vector(D::map("x", 10, 2), 2.0); + benchmark_join("small sparse vector multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, full_overlap) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::map("c", 16, 2), 2.0); + benchmark_join("sparse full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, full_overlap_big_vs_small) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); + auto rhs = make_cube(D::map("a", 2, 1), D::map("b", 2, 1), D::map("c", 2, 1), 2.0); + benchmark_join("sparse full overlap big vs small multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, partial_overlap) { + auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::map("d", 8, 1), 1.0); + auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::map("d", 8, 2), 2.0); + benchmark_join("sparse partial overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(SparseJoin, no_overlap) { + auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::map("f", 4, 1), 1.0); + auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::map("d", 4, 1), 2.0); + benchmark_join("sparse no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +TEST(MixedJoin, full_overlap) { + auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::idx("c", 16), 1.0); + auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::idx("c", 16), 2.0); + benchmark_join("mixed full overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(MixedJoin, partial_sparse_overlap) { + auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::idx("d", 8), 1.0); + auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::idx("d", 8), 2.0); + benchmark_join("mixed partial sparse overlap multiply", lhs, rhs, operation::Mul::f); +} + +TEST(MixedJoin, no_overlap) { + auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::idx("f", 4), 1.0); + auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::idx("d", 4), 2.0); + benchmark_join("mixed no overlap multiply", lhs, rhs, operation::Mul::f); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/packed_mappings/CMakeLists.txt b/eval/src/tests/tensor/packed_mappings/CMakeLists.txt new file mode 100644 index 00000000000..2d11755a0c5 --- /dev/null +++ b/eval/src/tests/tensor/packed_mappings/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(eval_packed_mappings_test_app TEST + SOURCES + packed_mappings_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_packed_mappings_test_app COMMAND eval_packed_mappings_test_app) + +vespa_add_executable(eval_packed_mixed_test_app TEST + SOURCES + packed_mixed_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_packed_mixed_test_app COMMAND eval_packed_mixed_test_app) diff --git a/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp new file mode 100644 index 00000000000..c8814372bf5 --- /dev/null +++ b/eval/src/tests/tensor/packed_mappings/packed_mappings_test.cpp @@ -0,0 +1,226 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/tensor/mixed/packed_labels.h> +#include <vespa/eval/tensor/mixed/packed_mappings.h> +#include <vespa/eval/tensor/mixed/packed_mappings_builder.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <stdlib.h> +#include <assert.h> +#include <set> + +using namespace vespalib::eval; +using namespace vespalib::eval::packed_mixed_tensor; + +namespace { + +uint32_t random_range(uint32_t from, uint32_t to) { + assert(from + 1 < to); + int unif = rand() % (to - from); + return from + unif; +} + +const char *mixed_tensor_types[] = { + "tensor<float>(x{})", + "tensor<float>(a{},b{},c{},d{},e{},f{})", + "tensor<float>(x{},y{})", + "tensor<float>(x{},z[3])", + "tensor<float>(w[5],x{},y{},z[3])" +}; + +const char *float_tensor_types[] = { + "tensor<float>(x{})", + "tensor<float>(x{},y{})", + "tensor<float>(x{},z[3])", + "tensor<float>(w[5],x{},y{},z[3])", + "tensor<float>(z[2])", + "tensor<float>()" +}; + + vespalib::string label1(""), + label2("foo"), + label3("bar"); + vespalib::string label4("foobar"), + label5("barfoo"), + label6("other"); + vespalib::string label7("long text number one"), + label8("long text number two"), + label9("long text number three"); + +std::vector<vespalib::stringref> +generate_random_address(uint32_t dims) +{ + std::vector<vespalib::stringref> foo(dims, label1); + for (auto & ref : foo) { + size_t pct = random_range(0, 100); + if (pct < 5) { ref = label1; } + else if (pct < 30) { ref = label2; } + else if (pct < 55) { ref = label3; } + else if (pct < 65) { ref = label4; } + else if (pct < 75) { ref = label5; } + else if (pct < 85) { ref = label6; } + else if (pct < 90) { ref = label7; } + else if (pct < 95) { ref = label8; } + else { ref = label9; } + } + return foo; +} + +} // namespace <unnamed> + +class MappingsBuilderTest : public ::testing::Test { +public: + std::unique_ptr<PackedMappingsBuilder> builder; + std::unique_ptr<PackedMappings> built; + + MappingsBuilderTest() = default; + + virtual ~MappingsBuilderTest() = default; + + void build_and_compare() { + ASSERT_TRUE(builder); + built = builder->build_mappings(); + ASSERT_TRUE(built); + EXPECT_EQ(builder->num_mapped_dims(), built->num_mapped_dims()); + EXPECT_EQ(builder->size(), built->size()); + for (size_t idx = 0; idx < built->size(); ++idx) { + std::vector<vespalib::stringref> got(builder->num_mapped_dims()); + built->fill_address_by_sortid(idx, got); + printf("Got address:"); + for (auto ref : got) { + printf(" '%s'", ref.data()); + } + uint32_t subspace = built->subspace_of_address(got); + uint32_t original = builder->add_mapping_for(got); + printf(" -> %u\n", original); + EXPECT_EQ(subspace, original); + } + } +}; + +TEST_F(MappingsBuilderTest, empty_mapping) +{ + for (uint32_t dims : { 0, 1, 2, 3 }) { + builder = std::make_unique<PackedMappingsBuilder>(dims); + build_and_compare(); + } +} + +TEST_F(MappingsBuilderTest, just_one) +{ + vespalib::string label("foobar"); + for (uint32_t dims : { 0, 1, 2, 3, 7 }) { + builder = std::make_unique<PackedMappingsBuilder>(dims); + std::vector<vespalib::stringref> foo(dims, label); + uint32_t idx = builder->add_mapping_for(foo); + EXPECT_EQ(idx, 0); + build_and_compare(); + } +} + +TEST_F(MappingsBuilderTest, some_random) +{ + for (uint32_t dims : { 1, 2, 5 }) { + builder = std::make_unique<PackedMappingsBuilder>(dims); + uint32_t cnt = random_range(dims*5, dims*20); + printf("Generate %u addresses for %u dims\n", cnt, dims); + for (uint32_t i = 0; i < cnt; ++i) { + auto foo = generate_random_address(dims); + uint32_t idx = builder->add_mapping_for(foo); + EXPECT_LE(idx, i); + } + build_and_compare(); + } +} + +class MixedBuilderTest : public ::testing::Test { +public: + std::unique_ptr<PackedMixedTensorBuilder<float>> builder; + std::unique_ptr<Value> built; + + MixedBuilderTest() = default; + + virtual ~MixedBuilderTest() = default; + + size_t expected_value = 0; + + void build_and_compare(size_t expect_size) { + built.reset(nullptr); + EXPECT_FALSE(built); + ASSERT_TRUE(builder); + built = builder->build(std::move(builder)); + EXPECT_FALSE(builder); + ASSERT_TRUE(built); + EXPECT_EQ(built->index().size(), expect_size); + auto cells = built->cells().typify<float>(); + for (float f : cells) { + float expect = ++expected_value; + EXPECT_EQ(f, expect); + } + } +}; + +TEST_F(MixedBuilderTest, empty_mapping) +{ + for (auto type_spec : mixed_tensor_types) { + ValueType type = ValueType::from_spec(type_spec); + size_t dims = type.count_mapped_dimensions(); + size_t dsss = type.dense_subspace_size(); + EXPECT_GT(dims, 0); + EXPECT_GT(dsss, 0); + builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); + build_and_compare(0); + } +} + +TEST_F(MixedBuilderTest, just_one) +{ + size_t counter = 0; + for (auto type_spec : float_tensor_types) { + ValueType type = ValueType::from_spec(type_spec); + size_t dims = type.count_mapped_dimensions(); + size_t dsss = type.dense_subspace_size(); + EXPECT_GT(dsss, 0); + builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); + auto address = generate_random_address(dims); + auto ref = builder->add_subspace(address); + EXPECT_EQ(ref.size(), dsss); + for (size_t i = 0; i < ref.size(); ++i) { + ref[i] = ++counter; + } + build_and_compare(1); + } +} + +TEST_F(MixedBuilderTest, some_random) +{ + size_t counter = 0; + for (auto type_spec : mixed_tensor_types) { + ValueType type = ValueType::from_spec(type_spec); + uint32_t dims = type.count_mapped_dimensions(); + uint32_t dsss = type.dense_subspace_size(); + EXPECT_GT(dims, 0); + EXPECT_GT(dsss, 0); + builder = std::make_unique<PackedMixedTensorBuilder<float>>(type, dims, dsss, 3); + + uint32_t cnt = random_range(dims*5, dims*20); + printf("MixBuild: generate %u addresses for %u dims\n", cnt, dims); + std::set<std::vector<vespalib::stringref>> seen; + for (uint32_t i = 0; i < cnt; ++i) { + auto address = generate_random_address(dims); + if (seen.insert(address).second) { + auto ref = builder->add_subspace(address); + EXPECT_EQ(ref.size(), dsss); + for (size_t j = 0; j < ref.size(); ++j) { + ref[j] = ++counter; + } + } + } + printf("MixBuild: generated %zu unique addresses\n", seen.size()); + build_and_compare(seen.size()); + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp new file mode 100644 index 00000000000..bc1efdaba1d --- /dev/null +++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp @@ -0,0 +1,153 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +std::vector<Layout> layouts = { + {}, + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + +TEST(PackedMixedTest, packed_mixed_tensors_can_be_converted_from_and_to_tensor_spec) { + for (const auto &layout: layouts) { + TensorSpec expect = spec(layout, N()); + std::unique_ptr<Value> value = value_from_spec(expect, PackedMixedTensorBuilderFactory::get()); + TensorSpec actual = spec_from_value(*value); + EXPECT_EQ(actual, expect); + } +} + +TEST(PackedMixedTest, packed_mixed_tensors_can_be_built_and_inspected) { + ValueType type = ValueType::from_spec("tensor<float>(x{},y[2],z{})"); + const auto & factory = PackedMixedTensorBuilderFactory::get(); + std::unique_ptr<ValueBuilder<float>> builder = factory.create_value_builder<float>(type); + float seq = 0.0; + for (vespalib::string x: {"a", "b", "c"}) { + for (vespalib::string y: {"aa", "bb"}) { + auto subspace = builder->add_subspace({x, y}); + EXPECT_EQ(subspace.size(), 2); + subspace[0] = seq + 1.0; + subspace[1] = seq + 5.0; + seq += 10.0; + } + seq += 100.0; + } + std::unique_ptr<Value> value = builder->build(std::move(builder)); + EXPECT_EQ(value->index().size(), 6); + auto view = value->index().create_view({0}); + vespalib::stringref query = "b"; + vespalib::stringref label; + size_t subspace; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "aa"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "bb"); + EXPECT_EQ(subspace, 3); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "c"; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "aa"); + EXPECT_EQ(subspace, 4); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "bb"); + EXPECT_EQ(subspace, 5); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "notpresent"; + view->lookup({&query}); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + view = value->index().create_view({1}); + query = "aa"; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "a"); + EXPECT_EQ(subspace, 0); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "b"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "c"); + EXPECT_EQ(subspace, 4); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "bb"; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "a"); + EXPECT_EQ(subspace, 1); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "b"); + EXPECT_EQ(subspace, 3); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "c"); + EXPECT_EQ(subspace, 5); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + query = "notpresent"; + view->lookup({&query}); + EXPECT_FALSE(view->next_result({&label}, subspace)); + + view = value->index().create_view({0,1}); + vespalib::stringref query_x = "b"; + vespalib::stringref query_y = "bb"; + view->lookup({&query_x, &query_y}); + EXPECT_TRUE(view->next_result({}, subspace)); + EXPECT_EQ(subspace, 3); + EXPECT_FALSE(view->next_result({}, subspace)); + + view = value->index().create_view({}); + vespalib::stringref label_x; + vespalib::stringref label_y; + view->lookup({}); + + const std::vector<vespalib::stringref*> out({&label_x, &label_y}); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "a"); + EXPECT_EQ(label_y, "aa"); + EXPECT_EQ(subspace, 0); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "a"); + EXPECT_EQ(label_y, "bb"); + EXPECT_EQ(subspace, 1); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "b"); + EXPECT_EQ(label_y, "aa"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "b"); + EXPECT_EQ(label_y, "bb"); + EXPECT_EQ(subspace, 3); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "c"); + EXPECT_EQ(label_y, "aa"); + EXPECT_EQ(subspace, 4); + EXPECT_TRUE(view->next_result(out, subspace)); + EXPECT_EQ(label_x, "c"); + EXPECT_EQ(label_y, "bb"); + EXPECT_EQ(subspace, 5); + EXPECT_FALSE(view->next_result(out, subspace)); +} + + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index d1491e4f758..e1009969b43 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/objects/hexdump.h> #include <ostream> #include <vespa/eval/tensor/dense/dense_tensor_view.h> +#include <vespa/eval/eval/value_codec.h> using namespace vespalib::tensor; using vespalib::eval::TensorSpec; @@ -47,6 +48,24 @@ void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) { ASSERT_EQUAL(i, cells.size()); } +TensorSpec verify_new_value_serialized(const ExpBuffer &exp, const TensorSpec &spec) { + const auto &factory = vespalib::eval::SimpleValueBuilderFactory::get(); + auto new_value = vespalib::eval::value_from_spec(spec, factory); + auto new_value_spec = vespalib::eval::spec_from_value(*new_value); + nbostream actual; + vespalib::eval::encode_value(*new_value, actual); + ASSERT_EQUAL(exp, actual); + auto new_decoded = vespalib::eval::decode_value(actual, factory); + auto new_decoded_spec = vespalib::eval::spec_from_value(*new_decoded); + EXPECT_EQUAL(0u, actual.size()); + EXPECT_EQUAL(new_value_spec, new_decoded_spec); + if (new_value->type().is_dense()) { + TEST_DO(verify_cells_only<float>(exp, new_value_spec)); + TEST_DO(verify_cells_only<double>(exp, new_value_spec)); + } + return new_decoded_spec; +} + void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { auto &engine = DefaultTensorEngine::ref(); auto value = engine.from_spec(spec); @@ -62,6 +81,8 @@ void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) { TEST_DO(verify_cells_only<float>(exp, value_spec)); TEST_DO(verify_cells_only<double>(exp, value_spec)); } + auto new_value_spec = verify_new_value_serialized(exp, spec); + EXPECT_EQUAL(value_spec, new_value_spec); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt index 04f151f7ced..ee9a793bba0 100644 --- a/eval/src/vespa/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/CMakeLists.txt @@ -2,12 +2,14 @@ vespa_add_library(vespaeval SOURCES $<TARGET_OBJECTS:eval_eval> + $<TARGET_OBJECTS:eval_instruction> $<TARGET_OBJECTS:eval_eval_llvm> $<TARGET_OBJECTS:eval_eval_test> $<TARGET_OBJECTS:eval_eval_value_cache> $<TARGET_OBJECTS:eval_gp> $<TARGET_OBJECTS:eval_tensor> $<TARGET_OBJECTS:eval_tensor_dense> + $<TARGET_OBJECTS:eval_tensor_mixed> $<TARGET_OBJECTS:eval_tensor_serialization> $<TARGET_OBJECTS:eval_tensor_sparse> INSTALL lib64 diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 973245607de..84cebe8cfd0 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -28,6 +28,7 @@ vespa_add_library(eval_eval OBJECT tensor_nodes.cpp tensor_spec.cpp value.cpp + value_codec.cpp value_type.cpp value_type_spec.cpp visit_stuff.cpp diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index 121db4ffb6e..e9c82789991 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -7,6 +7,7 @@ #include "tensor_engine.h" #include "make_tensor_function.h" #include "compile_tensor_function.h" +#include "simple_tensor_engine.h" #include <vespa/vespalib/util/classname.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/vespalib/util/benchmark_timer.h> @@ -117,4 +118,26 @@ InterpretedFunction::detect_issues(const Function &function) return Function::Issues(std::move(checker.issues)); } +InterpretedFunction::EvalSingle::EvalSingle(Instruction op) + : _state(SimpleTensorEngine::ref()), + _op(op) +{ +} + +InterpretedFunction::EvalSingle::EvalSingle(const TensorEngine &engine, Instruction op) + : _state(engine), + _op(op) +{ +} + +const Value & +InterpretedFunction::EvalSingle::eval(const std::vector<Value::CREF> &stack) +{ + _state.stash.clear(); + _state.stack = stack; + _op.perform(_state); + assert(_state.stack.size() == 1); + return _state.stack.back(); +} + } diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h index fb67fcb0b74..f94e3144cc5 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.h +++ b/eval/src/vespa/eval/eval/interpreted_function.h @@ -102,6 +102,25 @@ public: const Value &eval(Context &ctx, const LazyParams ¶ms) const; double estimate_cost_us(const std::vector<double> ¶ms, double budget = 5.0) const; static Function::Issues detect_issues(const Function &function); + + /** + * This inner class is used for testing and benchmarking. It runs + * a single interpreted instruction in isolation. Note that + * instructions manipulating the program counter or resolving + * parameters may not be run in this way. Also note that the stack + * must contain exactly one value after the instruction is + * executed. If no tensor engine is specified, SimpleTensorEngine + * is used (typically for instructions ignoring the engine). + **/ + class EvalSingle { + private: + State _state; + Instruction _op; + public: + EvalSingle(Instruction op); + EvalSingle(const TensorEngine &engine, Instruction op); + const Value &eval(const std::vector<Value::CREF> &stack); + }; }; } diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 6ab7df34b2f..64b2b6f8865 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -336,7 +336,7 @@ public: View(const SimpleTensor &tensor, const IndexList &selector) : _less(selector), _refs() { - for (const auto &cell: tensor.cells()) { + for (const auto &cell: tensor.my_cells()) { _refs.emplace_back(cell); } std::sort(_refs.begin(), _refs.end(), _less); @@ -738,7 +738,7 @@ SimpleTensor::encode(const SimpleTensor &tensor, nbostream &output) Format format(meta); output.putInt1_4Bytes(format.tag); encode_type(output, format, tensor.type(), meta); - maybe_encode_num_blocks(output, meta, tensor.cells().size() / meta.block_size); + maybe_encode_num_blocks(output, meta, tensor.my_cells().size() / meta.block_size); View view(tensor, meta.mapped); for (auto block = view.first_range(); !block.empty(); block = view.next_range(block)) { encode_mapped_labels(output, meta, block.begin()->get().address); diff --git a/eval/src/vespa/eval/eval/simple_tensor.h b/eval/src/vespa/eval/eval/simple_tensor.h index d717838176b..28e11a4ffd3 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.h +++ b/eval/src/vespa/eval/eval/simple_tensor.h @@ -79,11 +79,13 @@ public: using join_fun_t = double (*)(double, double); SimpleTensor(); + TypedCells cells() const override { abort(); } + const Index &index() const override { abort(); } explicit SimpleTensor(double value); SimpleTensor(const ValueType &type_in, Cells cells_in); double as_double() const final override; const ValueType &type() const override { return _type; } - const Cells &cells() const { return _cells; } + const Cells &my_cells() const { return _cells; } std::unique_ptr<SimpleTensor> map(map_fun_t function) const; std::unique_ptr<SimpleTensor> reduce(Aggregator &aggr, const std::vector<vespalib::string> &dimensions) const; std::unique_ptr<SimpleTensor> rename(const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to) const; diff --git a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp index 6c2a0fcd53d..491a310dc0b 100644 --- a/eval/src/vespa/eval/eval/simple_tensor_engine.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor_engine.cpp @@ -63,7 +63,7 @@ SimpleTensorEngine::to_spec(const Value &value) const const auto &dimensions = value.type().dimensions(); with_simple(value, [&spec,&dimensions](const SimpleTensor &simple_tensor) { - for (const auto &cell: simple_tensor.cells()) { + for (const auto &cell: simple_tensor.my_cells()) { TensorSpec::Address addr; assert(cell.address.size() == dimensions.size()); for (size_t i = 0; i < cell.address.size(); ++i) { diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index e8ab26078e6..1ccf6f8cd25 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simple_value.h" -#include "tensor_spec.h" #include "inline_operation.h" #include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/util/visit_ranges.h> @@ -25,77 +24,7 @@ struct CreateSimpleValueBuilderBase { } }; -struct CreateValueFromTensorSpec { - template <typename T> static std::unique_ptr<NewValue> invoke(const ValueType &type, const TensorSpec &spec, const ValueBuilderFactory &factory) { - using SparseKey = std::vector<vespalib::stringref>; - using DenseMap = std::map<size_t,T>; - std::map<SparseKey,DenseMap> map; - for (const auto &entry: spec.cells()) { - SparseKey sparse_key; - size_t dense_key = 0; - for (const auto &dim: type.dimensions()) { - auto pos = entry.first.find(dim.name); - assert(pos != entry.first.end()); - assert(pos->second.is_mapped() == dim.is_mapped()); - if (dim.is_mapped()) { - sparse_key.emplace_back(pos->second.name); - } else { - dense_key = (dense_key * dim.size) + pos->second.index; - } - } - map[sparse_key][dense_key] = entry.second; - } - auto builder = factory.create_value_builder<T>(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size()); - for (const auto &entry: map) { - auto subspace = builder->add_subspace(entry.first); - for (const auto &cell: entry.second) { - subspace[cell.first] = cell.second; - } - } - return builder->build(std::move(builder)); - } -}; - -struct CreateTensorSpecFromValue { - template <typename T> static TensorSpec invoke(const NewValue &value) { - auto cells = value.cells().typify<T>(); - TensorSpec spec(value.type().to_spec()); - size_t subspace_id = 0; - size_t subspace_size = value.type().dense_subspace_size(); - std::vector<vespalib::stringref> labels(value.type().count_mapped_dimensions()); - std::vector<vespalib::stringref*> label_refs; - for (auto &label: labels) { - label_refs.push_back(&label); - } - auto view = value.index().create_view({}); - view->lookup({}); - while (view->next_result(label_refs, subspace_id)) { - size_t label_idx = 0; - TensorSpec::Address addr; - for (const auto &dim: value.type().dimensions()) { - if (dim.is_mapped()) { - addr.emplace(dim.name, labels[label_idx++]); - } - } - for (size_t i = 0; i < subspace_size; ++i) { - size_t dense_key = i; - for (auto dim = value.type().dimensions().rbegin(); - dim != value.type().dimensions().rend(); ++dim) - { - if (dim->is_indexed()) { - size_t label = dense_key % dim->size; - addr.emplace(dim->name, label).first->second = TensorSpec::Label(label); - dense_key /= dim->size; - } - } - spec.add(addr, cells[(subspace_size * subspace_id) + i]); - } - } - return spec; - } -}; - -class SimpleValueView : public NewValue::Index::View { +class SimpleValueView : public Value::Index::View { private: using Addr = std::vector<vespalib::string>; using Map = std::map<Addr,size_t>; @@ -168,79 +97,6 @@ public: } }; -// Contains various state needed to perform the sparse part (all -// mapped dimensions) of the join operation. Performs swapping of -// sparse indexes to ensure that we look up entries from the smallest -// index in the largest index. -struct SparseJoinState { - bool swapped; - const NewValue::Index &first_index; - const NewValue::Index &second_index; - const std::vector<size_t> &second_view_dims; - std::vector<vespalib::stringref> full_address; - std::vector<vespalib::stringref*> first_address; - std::vector<const vespalib::stringref*> address_overlap; - std::vector<vespalib::stringref*> second_only_address; - size_t lhs_subspace; - size_t rhs_subspace; - size_t &first_subspace; - size_t &second_subspace; - - SparseJoinState(const SparseJoinPlan &plan, const NewValue::Index &lhs, const NewValue::Index &rhs) - : swapped(rhs.size() < lhs.size()), - first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), - second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), - full_address(plan.sources.size()), - first_address(), address_overlap(), second_only_address(), - lhs_subspace(), rhs_subspace(), - first_subspace(swapped ? rhs_subspace : lhs_subspace), - second_subspace(swapped ? lhs_subspace : rhs_subspace) - { - auto first_source = swapped ? SparseJoinPlan::Source::RHS : SparseJoinPlan::Source::LHS; - for (size_t i = 0; i < full_address.size(); ++i) { - if (plan.sources[i] == SparseJoinPlan::Source::BOTH) { - first_address.push_back(&full_address[i]); - address_overlap.push_back(&full_address[i]); - } else if (plan.sources[i] == first_source) { - first_address.push_back(&full_address[i]); - } else { - second_only_address.push_back(&full_address[i]); - } - } - } - ~SparseJoinState(); -}; -SparseJoinState::~SparseJoinState() = default; - -// Treats all values as mixed tensors. Needs output cell type as well -// as input cell types since output cell type cannot always be -// directly inferred. -struct GenericJoin { - template <typename LCT, typename RCT, typename OCT, typename Fun> static std::unique_ptr<NewValue> - invoke(const NewValue &lhs, const NewValue &rhs, join_fun_t function, - const SparseJoinPlan &sparse_plan, const DenseJoinPlan &dense_plan, - const ValueType &res_type, const ValueBuilderFactory &factory) - { - Fun fun(function); - auto lhs_cells = lhs.cells().typify<LCT>(); - auto rhs_cells = rhs.cells().typify<RCT>(); - SparseJoinState state(sparse_plan, lhs.index(), rhs.index()); - auto builder = factory.create_value_builder<OCT>(res_type, sparse_plan.sources.size(), dense_plan.out_size, state.first_index.size()); - auto outer = state.first_index.create_view({}); - auto inner = state.second_index.create_view(state.second_view_dims); - outer->lookup({}); - while (outer->next_result(state.first_address, state.first_subspace)) { - inner->lookup(state.address_overlap); - while (inner->next_result(state.second_only_address, state.second_subspace)) { - OCT *dst = builder->add_subspace(state.full_address).begin(); - auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; - dense_plan.execute(dense_plan.lhs_size * state.lhs_subspace, dense_plan.rhs_size * state.rhs_subspace, join_cells); - } - } - return builder->build(std::move(builder)); - } -}; - } // namespace <unnamed> //----------------------------------------------------------------------------- @@ -269,7 +125,7 @@ SimpleValue::SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_ SimpleValue::~SimpleValue() = default; -std::unique_ptr<NewValue::Index::View> +std::unique_ptr<Value::Index::View> SimpleValue::create_view(const std::vector<size_t> &dims) const { return std::make_unique<SimpleValueView>(_index, dims, _num_mapped_dims); @@ -300,6 +156,9 @@ SimpleValueT<T>::add_subspace(const std::vector<vespalib::stringref> &addr) //----------------------------------------------------------------------------- +SimpleValueBuilderFactory::SimpleValueBuilderFactory() = default; +SimpleValueBuilderFactory SimpleValueBuilderFactory::_factory; + std::unique_ptr<ValueBuilderBase> SimpleValueBuilderFactory::create_value_builder_base(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in, size_t) const @@ -309,105 +168,4 @@ SimpleValueBuilderFactory::create_value_builder_base(const ValueType &type, //----------------------------------------------------------------------------- -DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() -{ - enum class Case { NONE, LHS, RHS, BOTH }; - Case prev_case = Case::NONE; - auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { - if (my_case == prev_case) { - assert(!loop_cnt.empty()); - loop_cnt.back() *= my_size; - } else { - loop_cnt.push_back(my_size); - lhs_stride.push_back(in_lhs); - rhs_stride.push_back(in_rhs); - prev_case = my_case; - } - }; - auto visitor = overload - { - [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, - [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, - [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } - }; - auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); - auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); - for (size_t i = loop_cnt.size(); i-- > 0; ) { - out_size *= loop_cnt[i]; - if (lhs_stride[i] != 0) { - lhs_stride[i] = lhs_size; - lhs_size *= loop_cnt[i]; - } - if (rhs_stride[i] != 0) { - rhs_stride[i] = rhs_size; - rhs_size *= loop_cnt[i]; - } - } -} - -DenseJoinPlan::~DenseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) - : sources(), lhs_overlap(), rhs_overlap() -{ - size_t lhs_idx = 0; - size_t rhs_idx = 0; - auto visitor = overload - { - [&](visit_ranges_first, const auto &) { - sources.push_back(Source::LHS); - ++lhs_idx; - }, - [&](visit_ranges_second, const auto &) { - sources.push_back(Source::RHS); - ++rhs_idx; - }, - [&](visit_ranges_both, const auto &, const auto &) { - sources.push_back(Source::BOTH); - lhs_overlap.push_back(lhs_idx++); - rhs_overlap.push_back(rhs_idx++); - } - }; - auto lhs_dims = lhs_type.mapped_dimensions(); - auto rhs_dims = rhs_type.mapped_dimensions(); - visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), - [](const auto &a, const auto &b){ return (a.name < b.name); }); -} - -SparseJoinPlan::~SparseJoinPlan() = default; - -//----------------------------------------------------------------------------- - -using JoinTypify = TypifyValue<TypifyCellType,operation::TypifyOp2>; - -std::unique_ptr<NewValue> new_join(const NewValue &a, const NewValue &b, join_fun_t function, const ValueBuilderFactory &factory) { - auto res_type = ValueType::join(a.type(), b.type()); - assert(!res_type.is_error()); - SparseJoinPlan sparse_plan(a.type(), b.type()); - DenseJoinPlan dense_plan(a.type(), b.type()); - return typify_invoke<4,JoinTypify,GenericJoin>(a.type().cell_type(), b.type().cell_type(), res_type.cell_type(), function, - a, b, function, sparse_plan, dense_plan, res_type, factory); -} - -//----------------------------------------------------------------------------- - -std::unique_ptr<NewValue> new_value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { - ValueType type = ValueType::from_spec(spec.type()); - assert(!type.is_error()); - return typify_invoke<1,TypifyCellType,CreateValueFromTensorSpec>(type.cell_type(), type, spec, factory); -} - -//----------------------------------------------------------------------------- - -TensorSpec spec_from_new_value(const NewValue &value) { - return typify_invoke<1,TypifyCellType,CreateTensorSpecFromValue>(value.type().cell_type(), value); -} - -//----------------------------------------------------------------------------- - } diff --git a/eval/src/vespa/eval/eval/simple_value.h b/eval/src/vespa/eval/eval/simple_value.h index 892dd6f1da6..af943b2904d 100644 --- a/eval/src/vespa/eval/eval/simple_value.h +++ b/eval/src/vespa/eval/eval/simple_value.h @@ -3,8 +3,6 @@ #pragma once #include "value.h" -#include "value_type.h" -#include <vespa/eval/tensor/dense/typed_cells.h> #include <vespa/vespalib/stllike/string.h> #include <vector> #include <map> @@ -18,129 +16,13 @@ class TensorSpec; using TypedCells = ::vespalib::tensor::TypedCells; /** - * Experimental interface layer that will be moved into Value when all - * existing implementations are able to implement it. This interface - * will try to unify scalars, dense tensors, sparse tensors and mixed - * tensors while also enabling operations to be implemented - * efficiently using this interface without having knowledge about the - * actual implementation. Baseline operations will treat all values as - * mixed tensors. Simplified and optimized variants may replace them - * as done today based on type knowledge. - * - * All values are expected to be separated into a continuous area - * storing cells as concatenated dense subspaces, and an index - * structure used to look up label combinations; mapping them into a - * set of dense subspaces. - **/ -struct NewValue : Value { - - // Root lookup structure for mapping labels to dense subspace indexes - struct Index { - - // A view able to look up dense subspace indexes from labels - // specifying a partial address for the dimensions given to - // create_view. A view is re-usable. Lookups are performed by - // calling the lookup function and lookup results are - // extracted using the next_result function. - struct View { - - // look up dense subspace indexes from labels specifying a - // partial address for the dimensions given to - // create_view. Results from the lookup is extracted using - // the next_result function. - virtual void lookup(const std::vector<const vespalib::stringref*> &addr) = 0; - - // Extract the next result (if any) from the previous - // lookup into the given partial address and index. Only - // the labels for the dimensions NOT specified in - // create_view will be extracted here. - virtual bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) = 0; - - virtual ~View() {} - }; - - // total number of mappings (equal to the number of dense subspaces) - virtual size_t size() const = 0; - - // create a view able to look up dense subspaces based on - // labels from a subset of the mapped dimensions. - virtual std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const = 0; - - virtual ~Index() {} - }; - virtual TypedCells cells() const = 0; - virtual const Index &index() const = 0; - virtual ~NewValue() {} -}; - -/** - * Tagging interface used as return type from factories before - * downcasting to actual builder with specialized cell type. - **/ -struct ValueBuilderBase { - virtual ~ValueBuilderBase() {} -}; - -/** - * Interface used to build a value one dense subspace at a - * time. Enables decoupling of what the value should contain from how - * to store the value. - **/ -template <typename T> -struct ValueBuilder : ValueBuilderBase { - // add a dense subspace for the given address (label for all - // mapped dimensions in canonical order). Note that previously - // returned subspaces will be invalidated when new subspaces are - // added. Also note that adding the same subspace multiple times - // is not allowed. - virtual ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) = 0; - - // Given the ownership of the builder itself, produce the newly - // created value. This means that builders can only be used once, - // it also means values can build themselves. - virtual std::unique_ptr<NewValue> build(std::unique_ptr<ValueBuilder> self) = 0; -}; - -/** - * Factory able to create appropriate value builders. We do not really - * care about the full mathematical type here, but it needs to be - * passed since it is exposed in the value api. The expected number of - * subspaces is also passed since it enables the builder to pre-size - * internal structures appropriately. Note that since we are not able - * to have virtual templated functions we need to cast the created - * builder. With interoperability between all values. - **/ -struct ValueBuilderFactory { - template <typename T> - std::unique_ptr<ValueBuilder<T>> create_value_builder(const ValueType &type, - size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const - { - assert(check_cell_type<T>(type.cell_type())); - auto base = create_value_builder_base(type, num_mapped_dims_in, subspace_size_in, expected_subspaces); - ValueBuilder<T> *builder = dynamic_cast<ValueBuilder<T>*>(base.get()); - assert(builder); - base.release(); - return std::unique_ptr<ValueBuilder<T>>(builder); - } - template <typename T> - std::unique_ptr<ValueBuilder<T>> create_value_builder(const ValueType &type) const - { - return create_value_builder<T>(type, type.count_mapped_dimensions(), type.dense_subspace_size(), 1); - } - virtual ~ValueBuilderFactory() {} -protected: - virtual std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, - size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const = 0; -}; - -/** * A simple implementation of a generic value that can also be used to * build new values. This class focuses on simplicity over speed and * is intended as a reference implementation that can also be used to * test the correctness of tensor operations as they are moved away * from the implementation of individual tensor classes. **/ -class SimpleValue : public NewValue, public NewValue::Index +class SimpleValue : public Value, public Value::Index { private: using Addr = std::vector<vespalib::string>; @@ -155,7 +37,7 @@ public: SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in); ~SimpleValue() override; const ValueType &type() const override { return _type; } - const NewValue::Index &index() const override { return *this; } + const Value::Index &index() const override { return *this; } size_t size() const override { return _index.size(); } std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; }; @@ -173,103 +55,25 @@ public: ~SimpleValueT() override; TypedCells cells() const override { return TypedCells(ConstArrayRef<T>(_cells)); } ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) override; - std::unique_ptr<NewValue> build(std::unique_ptr<ValueBuilder<T>> self) override { + std::unique_ptr<Value> build(std::unique_ptr<ValueBuilder<T>> self) override { ValueBuilder<T>* me = this; assert(me == self.get()); self.release(); - return std::unique_ptr<NewValue>(this); + return std::unique_ptr<Value>(this); } }; /** - * ValueBuilderFactory implementation for SimpleValue. + * ValueBuilderFactory implementation for SimpleValue. **/ -struct SimpleValueBuilderFactory : ValueBuilderFactory { - ~SimpleValueBuilderFactory() override {} -protected: +class SimpleValueBuilderFactory : public ValueBuilderFactory { +private: + SimpleValueBuilderFactory(); + static SimpleValueBuilderFactory _factory; std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const override; +public: + static const SimpleValueBuilderFactory &get() { return _factory; } }; -/** - * Plan for how to traverse two partially overlapping dense subspaces - * in parallel, identifying all matching cell index combinations, in - * the exact order the joined cells will be stored in the result. The - * plan can be made up-front during tensor function compilation. - **/ -struct DenseJoinPlan { - size_t lhs_size; - size_t rhs_size; - size_t out_size; - std::vector<size_t> loop_cnt; - std::vector<size_t> lhs_stride; - std::vector<size_t> rhs_stride; - DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); - ~DenseJoinPlan(); - template <typename F> void execute(size_t lhs, size_t rhs, F &&f) const { - switch(loops_left(0)) { - case 0: return execute_few<F, 0>(0, lhs, rhs, std::forward<F>(f)); - case 1: return execute_few<F, 1>(0, lhs, rhs, std::forward<F>(f)); - case 2: return execute_few<F, 2>(0, lhs, rhs, std::forward<F>(f)); - case 3: return execute_few<F, 3>(0, lhs, rhs, std::forward<F>(f)); - default: return execute_many<F>(0, lhs, rhs, std::forward<F>(f)); - } - } -private: - size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } - template <typename F, size_t N> void execute_few(size_t idx, size_t lhs, size_t rhs, F &&f) const { - if constexpr (N == 0) { - f(lhs, rhs); - } else { - for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { - execute_few<F, N - 1>(idx + 1, lhs, rhs, std::forward<F>(f)); - } - } - } - template <typename F> void execute_many(size_t idx, size_t lhs, size_t rhs, F &&f) const { - for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { - if (loops_left(idx + 1) == 3) { - execute_few<F, 3>(idx + 1, lhs, rhs, std::forward<F>(f)); - } else { - execute_many<F>(idx + 1, lhs, rhs, std::forward<F>(f)); - } - } - } -}; - -/** - * Plan for how to join the sparse part (all mapped dimensions) - * between two values. The plan can be made up-front during tensor - * function compilation. - **/ -struct SparseJoinPlan { - enum class Source { LHS, RHS, BOTH }; - std::vector<Source> sources; - std::vector<size_t> lhs_overlap; - std::vector<size_t> rhs_overlap; - SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); - ~SparseJoinPlan(); -}; - -/** - * Generic join operation treating both values as mixed - * tensors. Packaging will change, and while the baseline join will - * not have information about low-level value class implementations, - * it will have up-front knowledge about types, specifically - * dimensional overlap and result type. - **/ -using join_fun_t = double (*)(double, double); -std::unique_ptr<NewValue> new_join(const NewValue &a, const NewValue &b, join_fun_t function, const ValueBuilderFactory &factory); - -/** - * Make a value from a tensor spec using a value builder factory - * interface, making it work with any value implementation. - **/ -std::unique_ptr<NewValue> new_value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory); - -/** - * Convert a generic value to a tensor spec. - **/ -TensorSpec spec_from_new_value(const NewValue &value); - } diff --git a/eval/src/vespa/eval/eval/value.cpp b/eval/src/vespa/eval/eval/value.cpp index 3629a5ad698..283950a6a67 100644 --- a/eval/src/vespa/eval/eval/value.cpp +++ b/eval/src/vespa/eval/eval/value.cpp @@ -6,6 +6,40 @@ namespace vespalib { namespace eval { +namespace { + +struct TrivialView : Value::Index::View { + bool first = false; + void lookup(const std::vector<const vespalib::stringref*> &) override { first = true; } + bool next_result(const std::vector<vespalib::stringref*> &, size_t &idx_out) override { + if (first) { + idx_out = 0; + first = false; + return true; + } else { + return false; + } + } +}; + +} // <unnamed> + + +TrivialIndex::TrivialIndex() = default; +TrivialIndex TrivialIndex::_index; + +size_t +TrivialIndex::size() const +{ + return 1; +} + +std::unique_ptr<Value::Index::View> +TrivialIndex::create_view(const std::vector<size_t> &) const +{ + return std::make_unique<TrivialView>(); +} + ValueType DoubleValue::_type = ValueType::double_type(); } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index cad76c93c5c..20923fcd621 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -3,7 +3,10 @@ #pragma once #include "value_type.h" +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/traits.h> +#include <vector> #include <memory> namespace vespalib::eval { @@ -14,15 +17,71 @@ class Tensor; * An abstract Value. **/ struct Value { - typedef std::unique_ptr<Value> UP; - typedef std::reference_wrapper<const Value> CREF; + using UP = std::unique_ptr<Value>; + using CREF = std::reference_wrapper<const Value>; + using TypedCells = tensor::TypedCells; + virtual const ValueType &type() const = 0; + virtual ~Value() {} + +// ---- new interface enabling separation of values and operations + // Root lookup structure for mapping labels to dense subspace indexes + struct Index { + + // A view able to look up dense subspace indexes from labels + // specifying a partial address for the dimensions given to + // create_view. A view is re-usable. Lookups are performed by + // calling the lookup function and lookup results are + // extracted using the next_result function. + struct View { + + // look up dense subspace indexes from labels specifying a + // partial address for the dimensions given to + // create_view. Results from the lookup is extracted using + // the next_result function. + virtual void lookup(const std::vector<const vespalib::stringref*> &addr) = 0; + + // Extract the next result (if any) from the previous + // lookup into the given partial address and index. Only + // the labels for the dimensions NOT specified in + // create_view will be extracted here. + virtual bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) = 0; + + virtual ~View() {} + }; + + // total number of mappings (equal to the number of dense subspaces) + virtual size_t size() const = 0; + + // create a view able to look up dense subspaces based on + // labels from a subset of the mapped dimensions. + virtual std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const = 0; + + virtual ~Index() {} + }; + virtual TypedCells cells() const = 0; + virtual const Index &index() const = 0; +// --- end of new interface + +// --- old interface that may be (partially) removed in the future virtual bool is_double() const { return false; } virtual bool is_tensor() const { return false; } virtual double as_double() const { return 0.0; } bool as_bool() const { return (as_double() != 0.0); } virtual const Tensor *as_tensor() const { return nullptr; } - virtual const ValueType &type() const = 0; - virtual ~Value() {} +// --- end of old interface +}; + +/** + * Common index for values without any mapped dimensions. + **/ +class TrivialIndex : public Value::Index { +private: + TrivialIndex(); + static TrivialIndex _index; + size_t size() const override; + std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; +public: + static const TrivialIndex &get() { return _index; } }; class DoubleValue : public Value @@ -32,12 +91,74 @@ private: static ValueType _type; public: DoubleValue(double value) : _value(value) {} + TypedCells cells() const override { return TypedCells(ConstArrayRef<double>(&_value, 1)); } + const Index &index() const override { return TrivialIndex::get(); } bool is_double() const override { return true; } double as_double() const override { return _value; } const ValueType &type() const override { return _type; } static const ValueType &double_type() { return _type; } }; +/** + * Tagging interface used as return type from factories before + * downcasting to actual builder with specialized cell type. + **/ +struct ValueBuilderBase { + virtual ~ValueBuilderBase() {} +}; + +/** + * Interface used to build a value one dense subspace at a + * time. Enables decoupling of what the value should contain from how + * to store the value. + **/ +template <typename T> +struct ValueBuilder : ValueBuilderBase { + // add a dense subspace for the given address (label for all + // mapped dimensions in canonical order). Note that previously + // returned subspaces will be invalidated when new subspaces are + // added. Also note that adding the same subspace multiple times + // is not allowed. + virtual ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) = 0; + + // Given the ownership of the builder itself, produce the newly + // created value. This means that builders can only be used once, + // it also means values can build themselves. + virtual std::unique_ptr<Value> build(std::unique_ptr<ValueBuilder> self) = 0; +}; + +/** + * Factory able to create appropriate value builders. We do not really + * care about the full mathematical type here, but it needs to be + * passed since it is exposed in the value api. The expected number of + * subspaces is also passed since it enables the builder to pre-size + * internal structures appropriately. Note that since we are not able + * to have virtual templated functions we need to cast the created + * builder. With interoperability between all values. + **/ +struct ValueBuilderFactory { + template <typename T> + std::unique_ptr<ValueBuilder<T>> create_value_builder(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const + { + assert(check_cell_type<T>(type.cell_type())); + auto base = create_value_builder_base(type, num_mapped_dims_in, subspace_size_in, expected_subspaces); + ValueBuilder<T> *builder = dynamic_cast<ValueBuilder<T>*>(base.get()); + assert(builder); + base.release(); + return std::unique_ptr<ValueBuilder<T>>(builder); + } + template <typename T> + std::unique_ptr<ValueBuilder<T>> create_value_builder(const ValueType &type) const + { + return create_value_builder<T>(type, type.count_mapped_dimensions(), type.dense_subspace_size(), 1); + } + virtual ~ValueBuilderFactory() {} +protected: + virtual std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const = 0; +}; + } VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue); diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp new file mode 100644 index 00000000000..1755ce164ff --- /dev/null +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -0,0 +1,306 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "value_codec.h" +#include "tensor_spec.h" +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/typify.h> + +namespace vespalib::eval { + +namespace { + +using CellType = ValueType::CellType; +using IndexList = std::vector<size_t>; + +constexpr uint32_t DOUBLE_CELL_TYPE = 0; +constexpr uint32_t FLOAT_CELL_TYPE = 1; + +inline uint32_t cell_type_to_id(CellType cell_type) { + switch (cell_type) { + case CellType::DOUBLE: return DOUBLE_CELL_TYPE; + case CellType::FLOAT: return FLOAT_CELL_TYPE; + } + abort(); +} + +inline CellType id_to_cell_type(uint32_t id) { + switch (id) { + case DOUBLE_CELL_TYPE: return CellType::DOUBLE; + case FLOAT_CELL_TYPE: return CellType::FLOAT; + } + abort(); +} + +struct Format { + bool has_sparse; + bool has_dense; + bool with_cell_type; + uint32_t tag; + explicit Format(const ValueType &type) + : has_sparse(type.count_mapped_dimensions() > 0), + has_dense((type.count_indexed_dimensions() > 0) || !has_sparse), + with_cell_type(type.cell_type() != CellType::DOUBLE), + tag((has_sparse ? 0x1 : 0) | (has_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {} + explicit Format(uint32_t tag_in) + : has_sparse((tag_in & 0x1) != 0), + has_dense((tag_in & 0x2) != 0), + with_cell_type((tag_in & 0x4) != 0), + tag(tag_in) {} + ~Format() {} +}; + +void maybe_encode_cell_type(nbostream &output, const Format &format, CellType cell_type) { + if (format.with_cell_type) { + output.putInt1_4Bytes(cell_type_to_id(cell_type)); + } +} + +void encode_type(nbostream &output, const Format &format, const ValueType &type) { + maybe_encode_cell_type(output, format, type.cell_type()); + if (format.has_sparse) { + output.putInt1_4Bytes(type.count_mapped_dimensions()); + for (const auto & dim : type.dimensions()) { + if (dim.is_mapped()) { + output.writeSmallString(dim.name); + } + } + } + if (format.has_dense) { + output.putInt1_4Bytes(type.count_indexed_dimensions()); + for (const auto & dim : type.dimensions()) { + if (dim.is_indexed()) { + output.writeSmallString(dim.name); + output.putInt1_4Bytes(dim.size); + } + } + } +} + +void maybe_encode_num_blocks(nbostream &output, bool has_mapped_dims, size_t num_blocks) { + if (has_mapped_dims) { + output.putInt1_4Bytes(num_blocks); + } +} + +CellType maybe_decode_cell_type(nbostream &input, const Format &format) { + if (format.with_cell_type) { + return id_to_cell_type(input.getInt1_4Bytes()); + } + return CellType::DOUBLE; +} + +ValueType decode_type(nbostream &input, const Format &format) { + CellType cell_type = maybe_decode_cell_type(input, format); + std::vector<ValueType::Dimension> dim_list; + if (format.has_sparse) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name); + } + } + if (format.has_dense) { + size_t cnt = input.getInt1_4Bytes(); + for (size_t i = 0; i < cnt; ++i) { + vespalib::string name; + input.readSmallString(name); + dim_list.emplace_back(name, input.getInt1_4Bytes()); + } + } + return ValueType::tensor_type(std::move(dim_list), cell_type); +} + +size_t maybe_decode_num_blocks(nbostream &input, bool has_mapped_dims, const Format &format) { + if (has_mapped_dims || !format.has_dense) { + return input.getInt1_4Bytes(); + } + return 1; +} + +void encode_mapped_labels(nbostream &output, size_t num_mapped_dims, const std::vector<vespalib::stringref> &addr) { + for (size_t i = 0; i < num_mapped_dims; ++i) { + output.writeSmallString(addr[i]); + } +} + +void decode_mapped_labels(nbostream &input, size_t num_mapped_dims, std::vector<vespalib::stringref> &addr) { + for (size_t i = 0; i < num_mapped_dims; ++i) { + size_t strSize = input.getInt1_4Bytes(); + addr[i] = vespalib::stringref(input.peek(), strSize); + input.adjustReadPos(strSize); + } +} + + +template<typename T> +void decode_cells(nbostream &input, size_t num_cells, ArrayRef<T> dst) +{ + T value; + for (size_t i = 0; i < num_cells; ++i) { + input >> value; + dst[i] = value; + } +} + +struct DecodeState { + const ValueType &type; + const size_t subspace_size; + const size_t num_blocks; + const size_t num_mapped_dims; +}; + +struct ContentDecoder { + template<typename T> + static std::unique_ptr<Value> invoke(nbostream &input, const DecodeState &state, const ValueBuilderFactory &factory) { + std::vector<vespalib::stringref> address(state.num_mapped_dims); + auto builder = factory.create_value_builder<T>(state.type, state.num_mapped_dims, state.subspace_size, state.num_blocks); + for (size_t i = 0; i < state.num_blocks; ++i) { + decode_mapped_labels(input, state.num_mapped_dims, address); + auto block_cells = builder->add_subspace(address); + decode_cells(input, state.subspace_size, block_cells); + } + return builder->build(std::move(builder)); + } +}; + +struct CreateValueFromTensorSpec { + template <typename T> static std::unique_ptr<Value> invoke(const ValueType &type, const TensorSpec &spec, const ValueBuilderFactory &factory) { + using SparseKey = std::vector<vespalib::stringref>; + using DenseMap = std::map<size_t,T>; + std::map<SparseKey,DenseMap> map; + for (const auto &entry: spec.cells()) { + SparseKey sparse_key; + size_t dense_key = 0; + for (const auto &dim: type.dimensions()) { + auto pos = entry.first.find(dim.name); + assert(pos != entry.first.end()); + assert(pos->second.is_mapped() == dim.is_mapped()); + if (dim.is_mapped()) { + sparse_key.emplace_back(pos->second.name); + } else { + dense_key = (dense_key * dim.size) + pos->second.index; + } + } + map[sparse_key][dense_key] = entry.second; + } + // if spec is missing the required dense space, add it here: + if (map.empty() && type.count_mapped_dimensions() == 0) { + map[{}][0] = 0; + } + auto builder = factory.create_value_builder<T>(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size()); + for (const auto &entry: map) { + auto subspace = builder->add_subspace(entry.first); + for (const auto &cell: entry.second) { + subspace[cell.first] = cell.second; + } + } + return builder->build(std::move(builder)); + } +}; + +struct CreateTensorSpecFromValue { + template <typename T> static TensorSpec invoke(const Value &value) { + auto cells = value.cells().typify<T>(); + TensorSpec spec(value.type().to_spec()); + size_t subspace_id = 0; + size_t subspace_size = value.type().dense_subspace_size(); + std::vector<vespalib::stringref> labels(value.type().count_mapped_dimensions()); + std::vector<vespalib::stringref*> label_refs; + for (auto &label: labels) { + label_refs.push_back(&label); + } + auto view = value.index().create_view({}); + view->lookup({}); + while (view->next_result(label_refs, subspace_id)) { + size_t label_idx = 0; + TensorSpec::Address addr; + for (const auto &dim: value.type().dimensions()) { + if (dim.is_mapped()) { + addr.emplace(dim.name, labels[label_idx++]); + } + } + for (size_t i = 0; i < subspace_size; ++i) { + size_t dense_key = i; + for (auto dim = value.type().dimensions().rbegin(); + dim != value.type().dimensions().rend(); ++dim) + { + if (dim->is_indexed()) { + size_t label = dense_key % dim->size; + addr.emplace(dim->name, label).first->second = TensorSpec::Label(label); + dense_key /= dim->size; + } + } + spec.add(addr, cells[(subspace_size * subspace_id) + i]); + } + } + return spec; + } +}; + +struct EncodeState { + size_t num_mapped_dims; + size_t subspace_size; +}; + +struct ContentEncoder { + template<typename T> + static void invoke(const Value &value, const EncodeState &state, nbostream &output) { + std::vector<vespalib::stringref> address(state.num_mapped_dims); + std::vector<vespalib::stringref*> a_refs(state.num_mapped_dims);; + for (size_t i = 0; i < state.num_mapped_dims; ++i) { + a_refs[i] = &address[i]; + } + auto view = value.index().create_view({}); + view->lookup({}); + size_t subspace; + while (view->next_result(a_refs, subspace)) { + encode_mapped_labels(output, state.num_mapped_dims, address); + auto iter = value.cells().typify<T>().begin(); + iter += (subspace * state.subspace_size); + for (size_t i = 0; i < state.subspace_size; ++i) { + output << *iter++; + } + } + } +}; + +} // namespace <unnamed> + +void encode_value(const Value &value, nbostream &output) { + size_t num_mapped_dims = value.type().count_mapped_dimensions(); + size_t dense_subspace_size = value.type().dense_subspace_size(); + Format format(value.type()); + output.putInt1_4Bytes(format.tag); + encode_type(output, format, value.type()); + maybe_encode_num_blocks(output, (num_mapped_dims > 0), value.cells().size / dense_subspace_size); + EncodeState state{num_mapped_dims, dense_subspace_size}; + typify_invoke<1,TypifyCellType,ContentEncoder>(value.type().cell_type(), value, state, output); +} + +std::unique_ptr<Value> decode_value(nbostream &input, const ValueBuilderFactory &factory) { + Format format(input.getInt1_4Bytes()); + ValueType type = decode_type(input, format); + size_t num_mapped_dims = type.count_mapped_dimensions(); + size_t dense_subspace_size = type.dense_subspace_size(); + const size_t num_blocks = maybe_decode_num_blocks(input, (num_mapped_dims > 0), format); + DecodeState state{type, dense_subspace_size, num_blocks, num_mapped_dims}; + return typify_invoke<1,TypifyCellType,ContentDecoder>(type.cell_type(), input, state, factory); +} + +//----------------------------------------------------------------------------- + +std::unique_ptr<Value> value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { + ValueType type = ValueType::from_spec(spec.type()); + assert(!type.is_error()); + return typify_invoke<1,TypifyCellType,CreateValueFromTensorSpec>(type.cell_type(), type, spec, factory); +} + +TensorSpec spec_from_value(const Value &value) { + return typify_invoke<1,TypifyCellType,CreateTensorSpecFromValue>(value.type().cell_type(), value); +} + +//----------------------------------------------------------------------------- + + +} // namespace diff --git a/eval/src/vespa/eval/eval/value_codec.h b/eval/src/vespa/eval/eval/value_codec.h new file mode 100644 index 00000000000..3644f952a6c --- /dev/null +++ b/eval/src/vespa/eval/eval/value_codec.h @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "simple_value.h" +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { class nbostream; } + +namespace vespalib::eval { + +/** + * encode a value (which must support the new APIs) to binary format + **/ +void encode_value(const Value &value, nbostream &output); + +/** + * decode a value from binary format + **/ +std::unique_ptr<Value> decode_value(nbostream &input, const ValueBuilderFactory &factory); + +/** + * Make a value from a tensor spec using a value builder factory + * interface, making it work with any value implementation. + **/ +std::unique_ptr<Value> value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory); + +/** + * Convert a generic value to a tensor spec. + **/ +TensorSpec spec_from_value(const Value &value); + +} diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index 30a36fcba1d..4b776fc4bcd 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -174,6 +174,18 @@ ValueType::is_dense() const } size_t +ValueType::count_indexed_dimensions() const +{ + size_t cnt = 0; + for (const auto &dim : dimensions()) { + if (dim.is_indexed()) { + ++cnt; + } + } + return cnt; +} + +size_t ValueType::count_mapped_dimensions() const { size_t cnt = 0; diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 4199b3a3381..a0934de2704 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -60,6 +60,7 @@ public: bool is_tensor() const { return (_type == Type::TENSOR); } bool is_sparse() const; bool is_dense() const; + size_t count_indexed_dimensions() const; size_t count_mapped_dimensions() const; size_t dense_subspace_size() const; const std::vector<Dimension> &dimensions() const { return _dimensions; } diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt new file mode 100644 index 00000000000..e5aae50750d --- /dev/null +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_library(eval_instruction OBJECT + SOURCES + generic_join + generic_rename +) diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp new file mode 100644 index 00000000000..8a1a199effa --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -0,0 +1,232 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "generic_join.h" +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/vespalib/util/overload.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <cassert> + +namespace vespalib::eval::instruction { + +using State = InterpretedFunction::State; +using Instruction = InterpretedFunction::Instruction; + +namespace { + +//----------------------------------------------------------------------------- + +template <typename T, typename IN> uint64_t wrap_param(const IN &value_in) { + const T &value = value_in; + static_assert(sizeof(uint64_t) == sizeof(&value)); + return (uint64_t)&value; +} + +template <typename T> const T &unwrap_param(uint64_t param) { + return *((const T *)param); +} + +//----------------------------------------------------------------------------- + +struct JoinParam { + ValueType res_type; + SparseJoinPlan sparse_plan; + DenseJoinPlan dense_plan; + join_fun_t function; + const ValueBuilderFactory &factory; + JoinParam(const ValueType &lhs_type, const ValueType &rhs_type, + join_fun_t function_in, const ValueBuilderFactory &factory_in) + : res_type(ValueType::join(lhs_type, rhs_type)), + sparse_plan(lhs_type, rhs_type), + dense_plan(lhs_type, rhs_type), + function(function_in), + factory(factory_in) + { + assert(!res_type.is_error()); + } + ~JoinParam(); +}; +JoinParam::~JoinParam() = default; + +//----------------------------------------------------------------------------- + +// Contains various state needed to perform the sparse part (all +// mapped dimensions) of the join operation. Performs swapping of +// sparse indexes to ensure that we look up entries from the smallest +// index in the largest index. +struct SparseJoinState { + bool swapped; + const Value::Index &first_index; + const Value::Index &second_index; + const std::vector<size_t> &second_view_dims; + std::vector<vespalib::stringref> full_address; + std::vector<vespalib::stringref*> first_address; + std::vector<const vespalib::stringref*> address_overlap; + std::vector<vespalib::stringref*> second_only_address; + size_t lhs_subspace; + size_t rhs_subspace; + size_t &first_subspace; + size_t &second_subspace; + + SparseJoinState(const SparseJoinPlan &plan, const Value::Index &lhs, const Value::Index &rhs) + : swapped(rhs.size() < lhs.size()), + first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), + second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), + full_address(plan.sources.size()), + first_address(), address_overlap(), second_only_address(), + lhs_subspace(), rhs_subspace(), + first_subspace(swapped ? rhs_subspace : lhs_subspace), + second_subspace(swapped ? lhs_subspace : rhs_subspace) + { + auto first_source = swapped ? SparseJoinPlan::Source::RHS : SparseJoinPlan::Source::LHS; + for (size_t i = 0; i < full_address.size(); ++i) { + if (plan.sources[i] == SparseJoinPlan::Source::BOTH) { + first_address.push_back(&full_address[i]); + address_overlap.push_back(&full_address[i]); + } else if (plan.sources[i] == first_source) { + first_address.push_back(&full_address[i]); + } else { + second_only_address.push_back(&full_address[i]); + } + } + } + ~SparseJoinState(); +}; +SparseJoinState::~SparseJoinState() = default; + +/* +template <typename LCT, typename RCT, typename OCT, typename Fun> +void generic_join() +*/ + +template <typename LCT, typename RCT, typename OCT, typename Fun> +void my_generic_join_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<JoinParam>(param_in); + Fun fun(param.function); + const Value &lhs = state.peek(1); + const Value &rhs = state.peek(0); + auto lhs_cells = lhs.cells().typify<LCT>(); + auto rhs_cells = rhs.cells().typify<RCT>(); + SparseJoinState sparse(param.sparse_plan, lhs.index(), rhs.index()); + auto builder = param.factory.create_value_builder<OCT>(param.res_type, param.sparse_plan.sources.size(), param.dense_plan.out_size, sparse.first_index.size()); + auto outer = sparse.first_index.create_view({}); + auto inner = sparse.second_index.create_view(sparse.second_view_dims); + outer->lookup({}); + while (outer->next_result(sparse.first_address, sparse.first_subspace)) { + inner->lookup(sparse.address_overlap); + while (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { + OCT *dst = builder->add_subspace(sparse.full_address).begin(); + auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; + param.dense_plan.execute(param.dense_plan.lhs_size * sparse.lhs_subspace, param.dense_plan.rhs_size * sparse.rhs_subspace, join_cells); + } + } + auto &result = state.stash.create<std::unique_ptr<Value>>(builder->build(std::move(builder))); + const Value &result_ref = *(result.get()); + state.pop_pop_push(result_ref); +}; + +struct SelectGenericJoinOp { + template <typename LCT, typename RCT, typename OCT, typename Fun> static auto invoke() { + return my_generic_join_op<LCT,RCT,OCT,Fun>; + } +}; + +//----------------------------------------------------------------------------- + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() +{ + enum class Case { NONE, LHS, RHS, BOTH }; + Case prev_case = Case::NONE; + auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { + if (my_case == prev_case) { + assert(!loop_cnt.empty()); + loop_cnt.back() *= my_size; + } else { + loop_cnt.push_back(my_size); + lhs_stride.push_back(in_lhs); + rhs_stride.push_back(in_rhs); + prev_case = my_case; + } + }; + auto visitor = overload + { + [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, + [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, + [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } + }; + auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); + auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); + for (size_t i = loop_cnt.size(); i-- > 0; ) { + out_size *= loop_cnt[i]; + if (lhs_stride[i] != 0) { + lhs_stride[i] = lhs_size; + lhs_size *= loop_cnt[i]; + } + if (rhs_stride[i] != 0) { + rhs_stride[i] = rhs_size; + rhs_size *= loop_cnt[i]; + } + } +} + +DenseJoinPlan::~DenseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : sources(), lhs_overlap(), rhs_overlap() +{ + size_t lhs_idx = 0; + size_t rhs_idx = 0; + auto visitor = overload + { + [&](visit_ranges_first, const auto &) { + sources.push_back(Source::LHS); + ++lhs_idx; + }, + [&](visit_ranges_second, const auto &) { + sources.push_back(Source::RHS); + ++rhs_idx; + }, + [&](visit_ranges_both, const auto &, const auto &) { + sources.push_back(Source::BOTH); + lhs_overlap.push_back(lhs_idx++); + rhs_overlap.push_back(rhs_idx++); + } + }; + auto lhs_dims = lhs_type.mapped_dimensions(); + auto rhs_dims = rhs_type.mapped_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); +} + +SparseJoinPlan::~SparseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +} + +//----------------------------------------------------------------------------- + +namespace vespalib::eval::instruction { + +using JoinTypify = TypifyValue<TypifyCellType,operation::TypifyOp2>; + +Instruction +GenericJoin::make_instruction(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash) +{ + auto ¶m = stash.create<JoinParam>(lhs_type, rhs_type, function, factory); + auto fun = typify_invoke<4,JoinTypify,SelectGenericJoinOp>(lhs_type.cell_type(), rhs_type.cell_type(), param.res_type.cell_type(), function); + return Instruction(fun, wrap_param<JoinParam>(param)); +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/generic_join.h b/eval/src/vespa/eval/instruction/generic_join.h new file mode 100644 index 00000000000..25647452dff --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -0,0 +1,87 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/interpreted_function.h> + +namespace vespalib { class Stash; } +namespace vespalib::eval { struct ValueBuilderFactory; } + +namespace vespalib::eval::instruction { + +using join_fun_t = double (*)(double, double); + +//----------------------------------------------------------------------------- + +struct GenericJoin { + static InterpretedFunction::Instruction + make_instruction(const ValueType &lhs_type, const ValueType &rhs_type, join_fun_t function, + const ValueBuilderFactory &factory, Stash &stash); +}; + +//----------------------------------------------------------------------------- + +/** + * Plan for how to traverse two partially overlapping dense subspaces + * in parallel, identifying all matching cell index combinations, in + * the exact order the joined cells will be stored in the result. The + * plan can be made up-front during tensor function compilation. + **/ +struct DenseJoinPlan { + size_t lhs_size; + size_t rhs_size; + size_t out_size; + std::vector<size_t> loop_cnt; + std::vector<size_t> lhs_stride; + std::vector<size_t> rhs_stride; + DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~DenseJoinPlan(); + template <typename F> void execute(size_t lhs, size_t rhs, F &&f) const { + switch(loops_left(0)) { + case 0: return execute_few<F, 0>(0, lhs, rhs, std::forward<F>(f)); + case 1: return execute_few<F, 1>(0, lhs, rhs, std::forward<F>(f)); + case 2: return execute_few<F, 2>(0, lhs, rhs, std::forward<F>(f)); + case 3: return execute_few<F, 3>(0, lhs, rhs, std::forward<F>(f)); + default: return execute_many<F>(0, lhs, rhs, std::forward<F>(f)); + } + } +private: + size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } + template <typename F, size_t N> void execute_few(size_t idx, size_t lhs, size_t rhs, F &&f) const { + if constexpr (N == 0) { + f(lhs, rhs); + } else { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + execute_few<F, N - 1>(idx + 1, lhs, rhs, std::forward<F>(f)); + } + } + } + template <typename F> void execute_many(size_t idx, size_t lhs, size_t rhs, F &&f) const { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + if (loops_left(idx + 1) == 3) { + execute_few<F, 3>(idx + 1, lhs, rhs, std::forward<F>(f)); + } else { + execute_many<F>(idx + 1, lhs, rhs, std::forward<F>(f)); + } + } + } +}; + +/** + * Plan for how to join the sparse part (all mapped dimensions) + * between two values. The plan can be made up-front during tensor + * function compilation. + **/ +struct SparseJoinPlan { + enum class Source { LHS, RHS, BOTH }; + std::vector<Source> sources; + std::vector<size_t> lhs_overlap; + std::vector<size_t> rhs_overlap; + SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~SparseJoinPlan(); +}; + +//----------------------------------------------------------------------------- + +} // namespace diff --git a/eval/src/vespa/eval/instruction/generic_rename.cpp b/eval/src/vespa/eval/instruction/generic_rename.cpp new file mode 100644 index 00000000000..cda0aa6e89a --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_rename.cpp @@ -0,0 +1,197 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "generic_rename.h" +#include <vespa/eval/eval/value.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/util/typify.h> +#include <cassert> + +namespace vespalib::eval::instruction { + +using State = InterpretedFunction::State; +using Instruction = InterpretedFunction::Instruction; + +namespace { + +template <typename T, typename IN> uint64_t wrap_param(const IN &value_in) { + const T &value = value_in; + static_assert(sizeof(uint64_t) == sizeof(&value)); + return (uint64_t)&value; +} + +template <typename T> const T &unwrap_param(uint64_t param) { + return *((const T *)param); +} + +const vespalib::string & +find_rename(const vespalib::string & original, + const std::vector<vespalib::string> &from, + const std::vector<vespalib::string> &to) +{ + for (size_t i = 0; i < from.size(); ++i) { + if (original == from[i]) { + return to[i]; + } + } + return original; +} + +size_t +find_index_of(const vespalib::string & name, + const std::vector<ValueType::Dimension> & dims) +{ + for (size_t i = 0; i < dims.size(); ++i) { + if (name == dims[i].name) { + return i; + } + } + abort(); // should not happen +} + +struct RenameParam { + ValueType res_type; + SparseRenamePlan sparse_plan; + DenseRenamePlan dense_plan; + const ValueBuilderFactory &factory; + RenameParam(const ValueType &lhs_type, const ValueType &output_type, + const std::vector<vespalib::string> &rename_dimension_from, + const std::vector<vespalib::string> &rename_dimension_to, + const ValueBuilderFactory &factory_in) + : res_type(output_type), + sparse_plan(lhs_type, output_type, rename_dimension_from, rename_dimension_to), + dense_plan(lhs_type, output_type, rename_dimension_from, rename_dimension_to), + factory(factory_in) + { + assert(!res_type.is_error()); + assert(lhs_type.cell_type() == res_type.cell_type()); + } + ~RenameParam(); +}; +RenameParam::~RenameParam() = default; + +template <typename CT> +std::unique_ptr<Value> +generic_rename(const Value &a, + const SparseRenamePlan &sparse_plan, const DenseRenamePlan &dense_plan, + const ValueType &res_type, const ValueBuilderFactory &factory) +{ + auto cells = a.cells().typify<CT>(); + std::vector<vespalib::stringref> output_address(sparse_plan.mapped_dims); + std::vector<vespalib::stringref*> input_address; + for (size_t maps_to : sparse_plan.output_dimensions) { + input_address.push_back(&output_address[maps_to]); + } + auto builder = factory.create_value_builder<CT>(res_type, + sparse_plan.mapped_dims, + dense_plan.subspace_size, + a.index().size()); + auto view = a.index().create_view({}); + view->lookup({}); + size_t subspace; + while (view->next_result(input_address, subspace)) { + CT *dst = builder->add_subspace(output_address).begin(); + size_t input_offset = dense_plan.subspace_size * subspace; + auto copy_cells = [&](size_t input_idx) { *dst++ = cells[input_idx]; }; + dense_plan.execute(input_offset, copy_cells); + } + return builder->build(std::move(builder)); +} + +template <typename CT> +void my_generic_rename_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<RenameParam>(param_in); + const Value &a = state.peek(0); + auto res_value = generic_rename<CT>(a, param.sparse_plan, param.dense_plan, + param.res_type, param.factory); + auto &result = state.stash.create<std::unique_ptr<Value>>(std::move(res_value)); + const Value &result_ref = *(result.get()); + state.pop_push(result_ref); +} + +struct SelectGenericRenameOp { + template <typename CT> static auto invoke() { + return my_generic_rename_op<CT>; + } +}; + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +SparseRenamePlan::SparseRenamePlan(const ValueType &input_type, + const ValueType &output_type, + const std::vector<vespalib::string> &from, + const std::vector<vespalib::string> &to) + : output_dimensions() +{ + const auto in_dims = input_type.mapped_dimensions(); + const auto out_dims = output_type.mapped_dimensions(); + mapped_dims = in_dims.size(); + assert(mapped_dims == out_dims.size()); + for (const auto & dim : in_dims) { + const auto & renamed_to = find_rename(dim.name, from, to); + size_t index = find_index_of(renamed_to, out_dims); + assert(index < mapped_dims); + output_dimensions.push_back(index); + } + assert(output_dimensions.size() == mapped_dims); +} + +SparseRenamePlan::~SparseRenamePlan() = default; + +DenseRenamePlan::DenseRenamePlan(const ValueType &lhs_type, + const ValueType &output_type, + const std::vector<vespalib::string> &from, + const std::vector<vespalib::string> &to) + : loop_cnt(), + stride(), + subspace_size(output_type.dense_subspace_size()) +{ + assert (subspace_size == lhs_type.dense_subspace_size()); + const auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); + const auto out_dims = output_type.nontrivial_indexed_dimensions(); + size_t num_dense_dims = lhs_dims.size(); + assert(num_dense_dims == out_dims.size()); + std::vector<size_t> lhs_loopcnt(num_dense_dims); + std::vector<size_t> lhs_stride(num_dense_dims, 1); + size_t lhs_size = 1; + for (size_t i = num_dense_dims; i-- > 0; ) { + lhs_stride[i] = lhs_size; + lhs_loopcnt[i] = lhs_dims[i].size; + lhs_size *= lhs_loopcnt[i]; + } + assert(lhs_size == subspace_size); + size_t prev_index = num_dense_dims; + for (const auto & dim : out_dims) { + const auto & renamed_from = find_rename(dim.name, to, from); + size_t index = find_index_of(renamed_from, lhs_dims); + assert(index < num_dense_dims); + if (prev_index + 1 == index) { + assert(stride.back() == lhs_stride[index] * lhs_loopcnt[index]); + loop_cnt.back() *= lhs_loopcnt[index]; + stride.back() = lhs_stride[index]; + } else { + loop_cnt.push_back(lhs_loopcnt[index]); + stride.push_back(lhs_stride[index]); + } + prev_index = index; + } +} + +DenseRenamePlan::~DenseRenamePlan() = default; + +InterpretedFunction::Instruction +GenericRename::make_instruction(const ValueType &lhs_type, const ValueType &output_type, + const std::vector<vespalib::string> &rename_dimension_from, + const std::vector<vespalib::string> &rename_dimension_to, + const ValueBuilderFactory &factory, Stash &stash) +{ + auto ¶m = stash.create<RenameParam>(lhs_type, output_type, + rename_dimension_from, rename_dimension_to, + factory); + auto fun = typify_invoke<1,TypifyCellType,SelectGenericRenameOp>(output_type.cell_type()); + return Instruction(fun, wrap_param<RenameParam>(param)); +} + +} // namespace + diff --git a/eval/src/vespa/eval/instruction/generic_rename.h b/eval/src/vespa/eval/instruction/generic_rename.h new file mode 100644 index 00000000000..ca9f45bd341 --- /dev/null +++ b/eval/src/vespa/eval/instruction/generic_rename.h @@ -0,0 +1,77 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/vespalib/stllike/string.h> +#include <vector> + +namespace vespalib::eval { struct ValueBuilderFactory; } + +namespace vespalib::eval::instruction { + +struct DenseRenamePlan { + std::vector<size_t> loop_cnt; + std::vector<size_t> stride; + const size_t subspace_size; + DenseRenamePlan(const ValueType &lhs_type, + const ValueType &output_type, + const std::vector<vespalib::string> &from, + const std::vector<vespalib::string> &to); + ~DenseRenamePlan(); + template <typename F> void execute(size_t offset, F &&f) const { + switch(loops_left(0)) { + case 0: return execute_few<F, 0>(0, offset, std::forward<F>(f)); + case 1: return execute_few<F, 1>(0, offset, std::forward<F>(f)); + case 2: return execute_few<F, 2>(0, offset, std::forward<F>(f)); + case 3: return execute_few<F, 3>(0, offset, std::forward<F>(f)); + default: return execute_many<F>(0, offset, std::forward<F>(f)); + } + } +private: + size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } + + template <typename F, size_t N> void execute_few(size_t idx, size_t offset, F &&f) const { + if constexpr (N == 0) { + f(offset); + } else { + for (size_t i = 0; i < loop_cnt[idx]; ++i) { + execute_few<F, N - 1>(idx + 1, offset, std::forward<F>(f)); + offset += stride[idx]; + } + } + } + template <typename F> void execute_many(size_t idx, size_t offset, F &&f) const { + for (size_t i = 0; i < loop_cnt[idx]; ++i) { + if (loops_left(idx + 1) == 3) { + execute_few<F, 3>(idx + 1, offset, std::forward<F>(f)); + } else { + execute_many<F>(idx + 1, offset, std::forward<F>(f)); + } + offset += stride[idx]; + } + } +}; + +struct SparseRenamePlan { + size_t mapped_dims; + std::vector<size_t> output_dimensions; + SparseRenamePlan(const ValueType &input_type, + const ValueType &output_type, + const std::vector<vespalib::string> &from, + const std::vector<vespalib::string> &to); + ~SparseRenamePlan(); +}; + +//----------------------------------------------------------------------------- + +struct GenericRename { + static InterpretedFunction::Instruction + make_instruction(const ValueType &lhs_type, const ValueType &output_type, + const std::vector<vespalib::string> &rename_dimension_from, + const std::vector<vespalib::string> &rename_dimension_to, + const ValueBuilderFactory &factory, Stash &stash); +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h index cf3e2864a30..f69c068192a 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h @@ -28,6 +28,8 @@ public: const eval::ValueType &fast_type() const { return _typeRef; } const TypedCells &cellsRef() const { return _cellsRef; } + TypedCells cells() const override { return _cellsRef; } + const Index &index() const override { return eval::TrivialIndex::get(); } bool operator==(const DenseTensorView &rhs) const; CellsIterator cellsIterator() const { return CellsIterator(_typeRef, _cellsRef); } diff --git a/eval/src/vespa/eval/tensor/join_tensors.h b/eval/src/vespa/eval/tensor/join_tensors.h index aa493c23656..d66d0c1bf8e 100644 --- a/eval/src/vespa/eval/tensor/join_tensors.h +++ b/eval/src/vespa/eval/tensor/join_tensors.h @@ -18,8 +18,8 @@ joinTensors(const TensorImplType &lhs, Function &&func) { DirectSparseTensorBuilder - builder(lhs.combineDimensionsWith(rhs), lhs.cells()); - for (const auto &rhsCell : rhs.cells()) { + builder(lhs.combineDimensionsWith(rhs), lhs.my_cells()); + for (const auto &rhsCell : rhs.my_cells()) { builder.insertCell(rhsCell.first, rhsCell.second, func); } return builder.build(); @@ -36,8 +36,8 @@ joinTensorsNegated(const TensorImplType &lhs, Function &&func) { DirectSparseTensorBuilder - builder(lhs.combineDimensionsWith(rhs), lhs.cells()); - for (const auto &rhsCell : rhs.cells()) { + builder(lhs.combineDimensionsWith(rhs), lhs.my_cells()); + for (const auto &rhsCell : rhs.my_cells()) { builder.insertCell(rhsCell.first, -rhsCell.second, func); } return builder.build(); diff --git a/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt new file mode 100644 index 00000000000..ceded3a7380 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_library(eval_tensor_mixed OBJECT + SOURCES + packed_labels.cpp + packed_mappings.cpp + packed_mappings_builder.cpp + packed_mixed_tensor_builder_factory.cpp + packed_mixed_tensor.cpp + packed_mixed_tensor_builder.cpp +) diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp new file mode 100644 index 00000000000..a05497f6b68 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.cpp @@ -0,0 +1,56 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_labels.h" +#include <assert.h> + +namespace vespalib::eval::packed_mixed_tensor { + +int32_t +PackedLabels::find_label(vespalib::stringref to_find) const +{ + uint32_t lo = 0; + uint32_t hi = num_labels(); + while (lo < hi) { + uint32_t mid = (lo + hi) / 2; + if (get_label(mid) < to_find) { + lo = mid + 1; + } else { + hi = mid; + } + } + assert(lo == hi); + if (lo < num_labels() && get_label(lo) == to_find) { + return lo; + } + return -1; +} + +vespalib::stringref +PackedLabels::get_label(uint32_t index) const +{ + assert(index < num_labels()); + + uint32_t this_offset = _offsets[index]; + uint32_t next_offset = _offsets[index+1]; + auto p = &_label_store[this_offset]; + size_t sz = next_offset - this_offset - 1; + return vespalib::stringref(p, sz); +} + +void +PackedLabels::validate_labels(uint32_t num_labels) +{ + assert(num_labels == _offsets.size()-1); + for (uint32_t i = 0; i < num_labels; ++i) { + assert(_offsets[i] < _offsets[i+1]); + uint32_t last_byte_index = _offsets[i+1] - 1; + assert(last_byte_index < _label_store.size()); + assert(_label_store[last_byte_index] == 0); + } + assert(_label_store.size() == _offsets[num_labels]); + for (uint32_t i = 0; i+1 < num_labels; ++i) { + assert(get_label(i) < get_label(i+1)); + } +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_labels.h b/eval/src/vespa/eval/tensor/mixed/packed_labels.h new file mode 100644 index 00000000000..dac338448fb --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_labels.h @@ -0,0 +1,42 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/arrayref.h> + +namespace vespalib::eval::packed_mixed_tensor { + +/** + * Stores labels for sparse (mapped) tensor dimensions, + * where each unique label value is stored only once, + * and the values are sorted. References data that + * must be constant and owned by some object with + * enclosing lifetime. + **/ +class PackedLabels { +public: + PackedLabels(uint32_t num_labels, + ConstArrayRef<uint32_t> offsets, + ConstArrayRef<char> label_store) + : _offsets(offsets), + _label_store(label_store) + { + validate_labels(num_labels); + } + + uint32_t num_labels() const { return _offsets.size() - 1; } + + // returns -1 if the given label value cannot be found + int32_t find_label(vespalib::stringref value) const; + + vespalib::stringref get_label(uint32_t index) const; + +private: + const ConstArrayRef<uint32_t> _offsets; + const ConstArrayRef<char> _label_store; + + void validate_labels(uint32_t num_labels); +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp new file mode 100644 index 00000000000..226ac01ef71 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.cpp @@ -0,0 +1,130 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mappings.h" +#include <assert.h> + +namespace vespalib::eval::packed_mixed_tensor { + + +int32_t +PackedMappings::subspace_of_address(const Address &address) const +{ + int32_t idx = sortid_of_address(address); + if (idx < 0) { + return -1; + } + uint32_t internal_idx = idx; + assert (internal_idx < _num_mappings); + return subspace_of_sortid(internal_idx); +} + +int32_t +PackedMappings::subspace_of_enums(const InternalAddress &address) const +{ + int32_t idx = sortid_of_enums(address); + if (idx < 0) { + return -1; + } + uint32_t internal_idx = idx; + assert (internal_idx < _num_mappings); + return subspace_of_sortid(internal_idx); +} + +int32_t +PackedMappings::sortid_of_address(const Address &address) const +{ + if (_num_dims == 0) return 0; + assert(address.size() == _num_dims); + std::vector<uint32_t> to_find; + to_find.reserve(_num_dims); + for (const auto & label_value : address) { + int32_t label_idx = _label_store.find_label(label_value); + if (label_idx < 0) { + return -1; + } + to_find.push_back(label_idx); + } + return sortid_of_enums(to_find); +} + +int32_t +PackedMappings::sortid_of_enums(const InternalAddress &address) const +{ + if (_num_dims == 0) return 0; + assert(address.size() == _num_dims); + const uint32_t * to_find = &address[0]; + uint32_t lo = 0; + uint32_t hi = _num_mappings; + while (lo < hi) { + uint32_t mid = (lo + hi) / 2; + if (enums_compare(ptr_of_sortid(mid), to_find) < 0) { + lo = mid + 1; + } else { + hi = mid; + } + } + assert(lo == hi); + if ((lo < _num_mappings) && + (enums_compare(ptr_of_sortid(lo), to_find) == 0)) + { + return lo; + } + return -1; +} + +/** returns subspace_index */ +uint32_t +PackedMappings::fill_enums_by_sortid(uint32_t internal_index, InternalAddress &address) const +{ + assert(internal_index < _num_mappings); + uint32_t offset = offset_of_mapping_data(internal_index); + address.resize(_num_dims); + for (uint32_t i = 0; i < _num_dims; ++i) { + address[i] = _int_store[offset++]; + } + return _int_store[offset]; +} + +/** returns subspace_index */ +uint32_t +PackedMappings::fill_address_by_sortid(uint32_t internal_index, Address &address) const +{ + assert(internal_index < _num_mappings); + uint32_t offset = offset_of_mapping_data(internal_index); + address.resize(_num_dims); + for (uint32_t i = 0; i < _num_dims; ++i) { + uint32_t label_idx = _int_store[offset++]; + address[i] = _label_store.get_label(label_idx); + } + return _int_store[offset]; +} + +void +PackedMappings::validate() const +{ + assert((_num_mappings * (1 + _num_dims)) == _int_store.size()); + auto iter = _int_store.cbegin(); + std::vector<uint32_t> prev; + std::vector<uint32_t> next; + for (uint32_t i = 0; i < _num_mappings; ++i) { + next.clear(); + for (uint32_t j = 0; j < _num_dims; ++j) { + uint32_t label_index = *iter++; + next.push_back(label_index); + assert(label_index < _label_store.num_labels()); + } + if (_num_dims == 0) { + assert(next == prev); + assert(i == 0); + assert(_num_mappings == 1); + } else { + assert(prev < next); + } + std::swap(prev, next); + uint32_t subspace_index = *iter++; + assert(subspace_index < _num_mappings); + } + assert(iter == _int_store.cend()); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h new file mode 100644 index 00000000000..ce9ff066271 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings.h @@ -0,0 +1,103 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "packed_labels.h" +#include <vector> + +namespace vespalib::eval::packed_mixed_tensor { + +/** + * Mappings for sparse tensor dimensions. + * + * Each address (conceptually "array of string") + * maps to a "subspace" (currently in the + * order that addresses were added to a builder). + * + * Internally addresses are lexicographically + * sorted, and you can iterate over them in sort + * order with the fill_*() methods. + * + * (Note: we may want to change this so subspaces + * are always sorted by address, so the "subspace" + * index and the "sortid" index become equivalent). + * + * Allows using the internal label enumerations + * instead of working with strings all the time. + * + * NOTE: Making a copy of PackedMappings will not copy + * the underlying data, these must then stay alive + * and unchanged for the lifetime of the copy as well. + **/ +class PackedMappings { +public: + using Address = std::vector<vespalib::stringref>; + using InternalAddress = std::vector<uint32_t>; + + uint32_t size() const { return _num_mappings; } + uint32_t num_mapped_dims() const { return _num_dims; } + + // returns -1 if mapping does not contain address + int32_t subspace_of_enums(const InternalAddress &address) const; + int32_t subspace_of_address(const Address &address) const; + + /** returns "subspace" index */ + uint32_t fill_address_by_sortid(uint32_t sortid, Address &address) const; + uint32_t fill_enums_by_sortid(uint32_t sortid, InternalAddress &address) const; + + // mapping from label enum to stringref (and vice versa) + const PackedLabels & label_store() const { return _label_store; } +private: + PackedMappings(uint32_t num_dims, uint32_t num_mappings, + ConstArrayRef<uint32_t> int_store, + PackedLabels label_store) + : _num_dims(num_dims), + _num_mappings(num_mappings), + _int_store(int_store), + _label_store(label_store) + { + validate(); + } + friend class PackedMappingsBuilder; + + void validate() const; + + const uint32_t _num_dims; + const uint32_t _num_mappings; + /* + _int_store contains data corresponding to this model: + struct IntStore { + // sorted lexicographically by label_enums: + struct MappingData { + uint32_t label_enums[num_dims]; + uint32_t subspace_index; + } mappings[num_mappings]; + }; + */ + const ConstArrayRef<uint32_t> _int_store; + const PackedLabels _label_store; + + int enums_compare(const uint32_t *a, const uint32_t *b) const { + for (size_t i = 0; i < _num_dims; ++i) { + if (a[i] < b[i]) return -1; + if (a[i] > b[i]) return 1; + } + return 0; + } + + uint32_t offset_of_mapping_data(uint32_t idx) const { + return (idx * (1 + _num_dims)); + } + uint32_t subspace_of_sortid(uint32_t internal_index) const { + uint32_t offset = offset_of_mapping_data(internal_index); + return _int_store[offset + _num_dims]; + } + const uint32_t * ptr_of_sortid(uint32_t internal_index) const { + return &_int_store[offset_of_mapping_data(internal_index)]; + } + + int32_t sortid_of_address(const Address &address) const; + int32_t sortid_of_enums(const InternalAddress &address) const; +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp new file mode 100644 index 00000000000..fdfe5957a3f --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.cpp @@ -0,0 +1,116 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mappings_builder.h" +#include <assert.h> + +namespace vespalib::eval::packed_mixed_tensor { + +PackedMappingsBuilder::~PackedMappingsBuilder() = default; + +uint32_t +PackedMappingsBuilder::add_mapping_for(SparseAddress address) +{ + assert(address.size() == _num_dims); + for (auto & label_value : address) { + // store label string in our own set: + auto iter = _labels.insert(label_value).first; + label_value = *iter; + } + uint32_t next_index = _mappings.size(); + auto iter = _mappings.emplace(address, next_index).first; + return iter->second; +} + + +size_t +PackedMappingsBuilder::extra_memory() const +{ + size_t int_store_cnt = (2 + _num_dims) * _mappings.size(); + size_t int_store_size = int_store_cnt * sizeof(uint32_t); + size_t label_cnt = _labels.size(); + size_t label_offsets_size = (1 + label_cnt) * sizeof(uint32_t); + size_t label_bytes = 0; + for (const auto & label_value : _labels) { + label_bytes += (label_value.size() + 1); + } + size_t extra_size = int_store_size + label_offsets_size + label_bytes; + return extra_size; +} + +PackedMappings +PackedMappingsBuilder::target_memory(char *mem_start, char *mem_end) const +{ + size_t int_store_cnt = (1 + _num_dims) * _mappings.size(); + size_t int_store_size = int_store_cnt * sizeof(uint32_t); + size_t label_cnt = _labels.size(); + size_t label_offsets_size = (1 + label_cnt) * sizeof(uint32_t); + + size_t label_bytes = 0; + for (const auto & label_value : _labels) { + label_bytes += (label_value.size() + 1); + } + + ssize_t needs_sz = int_store_size + label_offsets_size + label_bytes; + ssize_t avail_sz = mem_end - mem_start; + assert(needs_sz <= avail_sz); + + uint32_t * int_store_mem = (uint32_t *) (void *) mem_start; + uint32_t * offsets_mem = (uint32_t *) (void *) (mem_start + int_store_size); + char * labels_mem = mem_start + int_store_size + label_offsets_size; + + ArrayRef<uint32_t> int_store_data(int_store_mem, int_store_cnt); + ArrayRef<uint32_t> label_offsets(offsets_mem, 1 + label_cnt); + ArrayRef<char> labels_data(labels_mem, label_bytes); + assert(labels_data.end() <= mem_end); + + size_t byte_idx = 0; + size_t label_num = 0; + for (const auto & label_value : _labels) { + label_offsets[label_num++] = byte_idx; + size_t len_with_zero = label_value.size() + 1; + memcpy(&labels_data[byte_idx], label_value.data(), len_with_zero); + byte_idx += len_with_zero; + } + assert(label_num == label_cnt); + label_offsets[label_num] = byte_idx; + + assert(labels_data.begin() + byte_idx == labels_data.end()); + + PackedLabels stored_labels(label_cnt, label_offsets, labels_data); + + size_t int_store_offset = 0; + for (const auto & kv : _mappings) { + const SparseAddress & k = kv.first; + uint32_t v = kv.second; + for (const auto & label_value : k) { + int32_t label_idx = stored_labels.find_label(label_value); + assert(label_idx >= 0); + assert(uint32_t(label_idx) < label_num); + int_store_data[int_store_offset++] = label_idx; + } + int_store_data[int_store_offset++] = v; + } + assert(int_store_offset == int_store_cnt); + + return PackedMappings(_num_dims, _mappings.size(), + int_store_data, stored_labels); +} + +std::unique_ptr<PackedMappings> +PackedMappingsBuilder::build_mappings() const +{ + size_t self_size = sizeof(PackedMappings); + size_t total_size = self_size + extra_memory(); + + char * mem = (char *) operator new(total_size); + auto self_data = target_memory(mem + self_size, mem + total_size); + + PackedMappings * built = new (mem) PackedMappings(self_data); + + return std::unique_ptr<PackedMappings>(built); +} + +} // namespace + + + diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h new file mode 100644 index 00000000000..bf2ae434275 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mappings_builder.h @@ -0,0 +1,63 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "packed_mappings.h" +#include <vespa/vespalib/stllike/string.h> +#include <map> +#include <memory> +#include <set> +#include <vector> + +namespace vespalib::eval::packed_mixed_tensor { + +/** + * Builder for PackedMappings. + * Copies label values in all addresses added + * and packs all resulting data into a block of memory + * held by the built object, usually part of a larger + * aggregating object by using target_memory() method. + **/ +class PackedMappingsBuilder { +public: + using SparseAddress = std::vector<vespalib::stringref>; + + PackedMappingsBuilder(uint32_t num_mapped_dims) + : _num_dims(num_mapped_dims), + _labels(), + _mappings() + {} + + ~PackedMappingsBuilder(); + + // returns a new index for new addresses + // may be called multiple times with same address, + // will then return the same index for that address. + uint32_t add_mapping_for(SparseAddress address); + + // how much extra memory is needed by target_memory + // not including sizeof(PackedMappings) + size_t extra_memory() const; + + // put data that PackedMappings can refer to in the given + // memory block, and return an object referring to it. + PackedMappings target_memory(char *mem_start, char *mem_end) const; + + // number of dimensions + uint32_t num_mapped_dims() const { return _num_dims; } + + // how many unique addresses have been added? + size_t size() const { return _mappings.size(); } + + // build a self-contained PackedMappings object; + // used for unit testing. + std::unique_ptr<PackedMappings> build_mappings() const; + +private: + uint32_t _num_dims; + std::set<vespalib::string> _labels; + using IndexMap = std::map<SparseAddress, uint32_t>; + IndexMap _mappings; +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp new file mode 100644 index 00000000000..43f0c27ffbd --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.cpp @@ -0,0 +1,213 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_tensor.h" + +namespace vespalib::eval::packed_mixed_tensor { + +/*********************************************************************************/ + +class PackedMixedTensorIndexView : public Value::Index::View +{ +private: + const PackedMappings& _mappings; + const std::vector<size_t> _view_dims; + std::vector<uint32_t> _lookup_enums; + std::vector<uint32_t> _full_enums; + size_t _index; + + size_t num_full_dims() const { return _mappings.num_mapped_dims(); } + size_t num_view_dims() const { return _view_dims.size(); } + size_t num_rest_dims() const { return num_full_dims() - num_view_dims(); } +public: + PackedMixedTensorIndexView(const PackedMappings& mappings, + const std::vector<size_t> &dims) + : _mappings(mappings), + _view_dims(dims), + _lookup_enums(), + _index(0) + { + _lookup_enums.reserve(num_view_dims()); + _full_enums.resize(num_full_dims()); + } + + void lookup(const std::vector<const vespalib::stringref*> &addr) override; + bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) override; + ~PackedMixedTensorIndexView() override = default; +}; + +void +PackedMixedTensorIndexView::lookup(const std::vector<const vespalib::stringref*> &addr) +{ + _index = 0; + assert(addr.size() == num_view_dims()); + _lookup_enums.clear(); + for (const vespalib::stringref * label_ptr : addr) { + int32_t label_enum = _mappings.label_store().find_label(*label_ptr); + if (label_enum < 0) { + // cannot match + _index = _mappings.size(); + break; + } + _lookup_enums.push_back(label_enum); + } +} + +bool +PackedMixedTensorIndexView::next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) +{ + assert(addr_out.size() == num_rest_dims()); + while (_index < _mappings.size()) { + idx_out = _mappings.fill_enums_by_sortid(_index++, _full_enums); + bool couldmatch = true; + size_t vd_idx = 0; + size_t ao_idx = 0; + for (size_t i = 0; i < num_full_dims(); ++i) { + if (vd_idx < num_view_dims()) { + size_t next_view_dim = _view_dims[vd_idx]; + if (i == next_view_dim) { + if (_lookup_enums[vd_idx] == _full_enums[i]) { + // match in this dimension + ++vd_idx; + continue; + } else { + // does not match + couldmatch = false; + break; + } + } + } + // not a view dimension: + uint32_t label_enum = _full_enums[i]; + *addr_out[ao_idx] = _mappings.label_store().get_label(label_enum); + ++ao_idx; + } + if (couldmatch) { + assert(vd_idx == num_view_dims()); + assert(ao_idx == num_rest_dims()); + return true; + } + } + return false; +} + +/*********************************************************************************/ + +class PackedMixedTensorLookup : public Value::Index::View +{ +private: + const PackedMappings& _mappings; + std::vector<uint32_t> _lookup_enums; + bool _first_time; + + size_t num_full_dims() const { return _mappings.num_mapped_dims(); } +public: + PackedMixedTensorLookup(const PackedMappings& mappings) + : _mappings(mappings), + _lookup_enums(), + _first_time(false) + { + _lookup_enums.reserve(num_full_dims()); + } + + void lookup(const std::vector<const vespalib::stringref*> &addr) override; + bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) override; + ~PackedMixedTensorLookup() override = default; +}; + +void +PackedMixedTensorLookup::lookup(const std::vector<const vespalib::stringref*> &addr) +{ + assert(addr.size() == num_full_dims()); + _lookup_enums.clear(); + for (const vespalib::stringref * label_ptr : addr) { + int32_t label_enum = _mappings.label_store().find_label(*label_ptr); + if (label_enum < 0) { + // cannot match + _first_time = false; + return; + } + _lookup_enums.push_back(label_enum); + } + _first_time = true; +} + +bool +PackedMixedTensorLookup::next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) +{ + assert(addr_out.size() == 0); + if (_first_time) { + _first_time = false; + int32_t subspace = _mappings.subspace_of_enums(_lookup_enums); + if (subspace >= 0) { + idx_out = subspace; + return true; + } + } + return false; +} + +/*********************************************************************************/ + +class PackedMixedTensorAllMappings : public Value::Index::View +{ +private: + const PackedMappings& _mappings; + std::vector<vespalib::stringref> _full_address; + size_t _index; + +public: + PackedMixedTensorAllMappings(const PackedMappings& mappings) + : _mappings(mappings), + _full_address(), + _index(0) + { + _full_address.resize(_mappings.num_mapped_dims()); + } + + void lookup(const std::vector<const vespalib::stringref*> &addr) override; + bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) override; + ~PackedMixedTensorAllMappings() override = default; +}; + +void +PackedMixedTensorAllMappings::lookup(const std::vector<const vespalib::stringref*> &addr) +{ + _index = 0; + assert(addr.size() == 0); +} + +bool +PackedMixedTensorAllMappings::next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) +{ + assert(addr_out.size() == _mappings.num_mapped_dims()); + while (_index < _mappings.size()) { + idx_out = _mappings.fill_address_by_sortid(_index++, _full_address); + for (size_t i = 0; i < _mappings.num_mapped_dims(); ++i) { + *addr_out[i] = _full_address[i]; + } + return true; + } + return false; +} + +/*********************************************************************************/ + +PackedMixedTensor::~PackedMixedTensor() = default; + +std::unique_ptr<Value::Index::View> +PackedMixedTensor::create_view(const std::vector<size_t> &dims) const +{ + if (dims.size() == 0) { + return std::make_unique<PackedMixedTensorAllMappings>(_mappings); + } + for (size_t i = 1; i < dims.size(); ++i) { + assert(dims[i-1] < dims[i]); + assert(dims[i] < _mappings.num_mapped_dims()); + } + if (dims.size() == _mappings.num_mapped_dims()) { + return std::make_unique<PackedMixedTensorLookup>(_mappings); + } + return std::make_unique<PackedMixedTensorIndexView>(_mappings, dims); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h new file mode 100644 index 00000000000..604b1c94aeb --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor.h @@ -0,0 +1,51 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/simple_value.h> + +#include <vespa/eval/tensor/mixed/packed_mappings.h> +#include <vespa/eval/tensor/mixed/packed_mappings_builder.h> + +namespace vespalib::eval::packed_mixed_tensor { + +/** + * An implementation of Value modeling a mixed tensor, + * where all the data (cells and sparse address mappings) + * can reside in a self-contained, contigous block of memory. + * Currently must be built by a PackedMixedTensorBuilder. + * Immutable (all data always const). + **/ +class PackedMixedTensor : public Value, public Value::Index +{ +private: + const ValueType _type; + const TypedCells _cells; + const PackedMappings _mappings; + + PackedMixedTensor(const ValueType &type, + TypedCells cells, + const PackedMappings &mappings) + : _type(type), + _cells(cells), + _mappings(mappings) + {} + + template<typename T> friend class PackedMixedTensorBuilder; + +public: + ~PackedMixedTensor() override; + + // Value API: + const ValueType &type() const override { return _type; } + const Value::Index &index() const override { return *this; } + TypedCells cells() const override { return _cells; } + + // Value::Index API: + size_t size() const override { return _mappings.size(); } + std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp new file mode 100644 index 00000000000..75b307b1aa7 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.cpp @@ -0,0 +1,53 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_tensor_builder.h" + +namespace vespalib::eval::packed_mixed_tensor { + +template <typename T> +ArrayRef<T> +PackedMixedTensorBuilder<T>::add_subspace(const std::vector<vespalib::stringref> &addr) +{ + uint32_t idx = _mappings_builder.add_mapping_for(addr); + size_t offset = idx * _subspace_size; + assert(offset <= _cells.size()); + if (offset == _cells.size()) { + _cells.resize(offset + _subspace_size); + } + return ArrayRef<T>(&_cells[offset], _subspace_size); +} + + +template <typename T> +std::unique_ptr<Value> +PackedMixedTensorBuilder<T>::build(std::unique_ptr<ValueBuilder<T>>) +{ + size_t self_size = sizeof(PackedMixedTensor); + size_t mappings_size = _mappings_builder.extra_memory(); + // align: + mappings_size += 15ul; + mappings_size &= ~15ul; + size_t cells_size = sizeof(T) * _cells.size(); + size_t total_size = self_size + mappings_size + cells_size; + + char *mem = (char *) operator new(total_size); + char *mappings_mem = mem + self_size; + char *cells_mem = mappings_mem + mappings_size; + + // fill mapping data: + auto mappings = _mappings_builder.target_memory(mappings_mem, cells_mem); + + // copy cells: + memcpy(cells_mem, &_cells[0], cells_size); + ConstArrayRef<T> cells((T *)(void *) cells_mem, _cells.size()); + + PackedMixedTensor * built = + new (mem) PackedMixedTensor(_type, TypedCells(cells), mappings); + + return std::unique_ptr<PackedMixedTensor>(built); +} + +template class PackedMixedTensorBuilder<float>; +template class PackedMixedTensorBuilder<double>; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h new file mode 100644 index 00000000000..c99762b7e8b --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder.h @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "packed_mixed_tensor.h" + +namespace vespalib::eval::packed_mixed_tensor { + +/** + * A builder for PackedMixedTensor objects + * appropriate for cell type T. + **/ +template <typename T> +class PackedMixedTensorBuilder : public ValueBuilder<T> +{ +private: + const ValueType & _type; + size_t _subspace_size; + std::vector<T> _cells; + PackedMappingsBuilder _mappings_builder; +public: + PackedMixedTensorBuilder(const ValueType &type, + size_t num_mapped_in, + size_t subspace_size_in, + size_t expected_subspaces) + : _type(type), + _subspace_size(subspace_size_in), + _cells(), + _mappings_builder(num_mapped_in) + { + _cells.reserve(_subspace_size * expected_subspaces); + } + + ~PackedMixedTensorBuilder() override = default; + + ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) override; + std::unique_ptr<Value> build(std::unique_ptr<ValueBuilder<T>> self) override; +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp new file mode 100644 index 00000000000..48eedd86f7f --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.cpp @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "packed_mixed_tensor_builder_factory.h" +#include "packed_mixed_tensor_builder.h" + +#include <vespa/vespalib/util/typify.h> + +namespace vespalib::eval { + +namespace { + +struct CreatePackedMixedTensorBuilder { + template <typename T, typename ...Args> + static std::unique_ptr<ValueBuilderBase> invoke(const ValueType &type, Args &&...args) + { + assert(check_cell_type<T>(type.cell_type())); + return std::make_unique<packed_mixed_tensor::PackedMixedTensorBuilder<T>>(type, std::forward<Args>(args)...); + } +}; + +} // namespace <unnamed> + +PackedMixedTensorBuilderFactory::PackedMixedTensorBuilderFactory() = default; +PackedMixedTensorBuilderFactory PackedMixedTensorBuilderFactory::_factory; + +std::unique_ptr<ValueBuilderBase> +PackedMixedTensorBuilderFactory::create_value_builder_base(const ValueType &type, + size_t num_mapped_in, + size_t subspace_size_in, + size_t expected_subspaces) const +{ + return typify_invoke<1,TypifyCellType,CreatePackedMixedTensorBuilder>(type.cell_type(), + type, num_mapped_in, subspace_size_in, expected_subspaces); +} + +} // namespace diff --git a/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h new file mode 100644 index 00000000000..20a581e2b35 --- /dev/null +++ b/eval/src/vespa/eval/tensor/mixed/packed_mixed_tensor_builder_factory.h @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/simple_value.h> + +namespace vespalib::eval { + +/** + * A factory that can generate PackedMixedTensorBuilder + * objects appropriate for the requested CellType. + */ +struct PackedMixedTensorBuilderFactory : ValueBuilderFactory { +private: + PackedMixedTensorBuilderFactory(); + static PackedMixedTensorBuilderFactory _factory; + ~PackedMixedTensorBuilderFactory() override {} +protected: + std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, + size_t num_mapped_in, size_t subspace_size_in, size_t expect_subspaces) const override; +public: + static const PackedMixedTensorBuilderFactory &get() { return _factory; } +}; + +} // namespace diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp index 1538ecdd12f..98a20cd9630 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp @@ -204,18 +204,18 @@ SparseTensor::merge(join_fun_t function, const Tensor &arg) const const SparseTensor *rhs = dynamic_cast<const SparseTensor *>(&arg); assert(rhs && (fast_type().dimensions() == rhs->fast_type().dimensions())); DirectSparseTensorBuilder builder(eval::ValueType::merge(fast_type(), rhs->fast_type())); - builder.reserve(cells().size() + rhs->cells().size()); - for (const auto &cell: cells()) { - auto pos = rhs->cells().find(cell.first); - if (pos == rhs->cells().end()) { + builder.reserve(my_cells().size() + rhs->my_cells().size()); + for (const auto &cell: my_cells()) { + auto pos = rhs->my_cells().find(cell.first); + if (pos == rhs->my_cells().end()) { builder.insertCell(cell.first, cell.second); } else { builder.insertCell(cell.first, function(cell.second, pos->second)); } } - for (const auto &cell: rhs->cells()) { - auto pos = cells().find(cell.first); - if (pos == cells().end()) { + for (const auto &cell: rhs->my_cells()) { + auto pos = my_cells().find(cell.first); + if (pos == my_cells().end()) { builder.insertCell(cell.first, cell.second); } } diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h index 34a6622bdd6..002e0dac0ef 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h @@ -34,8 +34,10 @@ private: public: explicit SparseTensor(const eval::ValueType &type_in, const Cells &cells_in); SparseTensor(eval::ValueType &&type_in, Cells &&cells_in, Stash &&stash_in); + TypedCells cells() const override { abort(); } + const Index &index() const override { abort(); } ~SparseTensor() override; - const Cells &cells() const { return _cells; } + const Cells &my_cells() const { return _cells; } const eval::ValueType &fast_type() const { return _type; } bool operator==(const SparseTensor &rhs) const; eval::ValueType combineDimensionsWith(const SparseTensor &rhs) const; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp index 9d619f0f41a..8d46e88ca72 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.hpp @@ -14,13 +14,13 @@ apply(const SparseTensor &lhs, const SparseTensor &rhs, Function &&func) { DirectSparseTensorBuilder builder(lhs.combineDimensionsWith(rhs)); TensorAddressCombiner addressCombiner(lhs.fast_type(), rhs.fast_type()); - size_t estimatedCells = (lhs.cells().size() * rhs.cells().size()); + size_t estimatedCells = (lhs.my_cells().size() * rhs.my_cells().size()); if (addressCombiner.numOverlappingDimensions() != 0) { - estimatedCells = std::min(lhs.cells().size(), rhs.cells().size()); + estimatedCells = std::min(lhs.my_cells().size(), rhs.my_cells().size()); } builder.reserve(estimatedCells*2); - for (const auto &lhsCell : lhs.cells()) { - for (const auto &rhsCell : rhs.cells()) { + for (const auto &lhsCell : lhs.my_cells()) { + for (const auto &rhsCell : rhs.my_cells()) { bool combineSuccess = addressCombiner.combine(lhsCell.first, rhsCell.first); if (combineSuccess) { builder.insertCell(addressCombiner.getAddressRef(), diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp index 75eff05cb37..9dc47b0176c 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp @@ -12,10 +12,10 @@ namespace vespalib::tensor { void SparseTensorMatch::fastMatch(const TensorImplType &lhs, const TensorImplType &rhs) { - _builder.reserve(lhs.cells().size()); - for (const auto &lhsCell : lhs.cells()) { - auto rhsItr = rhs.cells().find(lhsCell.first); - if (rhsItr != rhs.cells().end()) { + _builder.reserve(lhs.my_cells().size()); + for (const auto &lhsCell : lhs.my_cells()) { + auto rhsItr = rhs.my_cells().find(lhsCell.first); + if (rhsItr != rhs.my_cells().end()) { _builder.insertCell(lhsCell.first, lhsCell.second * rhsItr->second); } } @@ -28,7 +28,7 @@ SparseTensorMatch::SparseTensorMatch(const TensorImplType &lhs, const TensorImpl assert (lhs.fast_type().dimensions().size() == _builder.fast_type().dimensions().size()); // Ensure that first tensor to fastMatch has fewest cells. - if (lhs.cells().size() <= rhs.cells().size()) { + if (lhs.my_cells().size() <= rhs.my_cells().size()) { fastMatch(lhs, rhs); } else { fastMatch(rhs, lhs); diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp index 2016dc2207a..f55fec85155 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_reduce.hpp @@ -12,8 +12,8 @@ std::unique_ptr<Tensor> reduceAll(const SparseTensor &tensor, DirectSparseTensorBuilder &builder, Function &&func) { - auto itr = tensor.cells().begin(); - auto itrEnd = tensor.cells().end(); + auto itr = tensor.my_cells().begin(); + auto itrEnd = tensor.my_cells().end(); double result = 0.0; if (itr != itrEnd) { result = itr->second; @@ -47,8 +47,8 @@ reduce(const SparseTensor &tensor, return reduceAll(tensor, builder, func); } TensorAddressReducer addressReducer(tensor.fast_type(), dimensions); - builder.reserve(tensor.cells().size()*2); - for (const auto &cell : tensor.cells()) { + builder.reserve(tensor.my_cells().size()*2); + for (const auto &cell : tensor.my_cells()) { addressReducer.reduce(cell.first); builder.insertCell(addressReducer.getAddressRef(), cell.second, func); } diff --git a/eval/src/vespa/eval/tensor/tensor_apply.cpp b/eval/src/vespa/eval/tensor/tensor_apply.cpp index 8f0610fed65..98450797f0c 100644 --- a/eval/src/vespa/eval/tensor/tensor_apply.cpp +++ b/eval/src/vespa/eval/tensor/tensor_apply.cpp @@ -10,7 +10,7 @@ TensorApply<TensorT>::TensorApply(const TensorImplType &tensor, const CellFunction &func) : Parent(tensor.fast_type()) { - for (const auto &cell : tensor.cells()) { + for (const auto &cell : tensor.my_cells()) { _builder.insertCell(cell.first, func.apply(cell.second)); } } diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp index 08fbb945830..241b8026b59 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.cpp @@ -41,7 +41,7 @@ WrappedSimpleTensor::accept(TensorVisitor &visitor) const { TensorAddressBuilder addr; const auto &dimensions = _tensor.type().dimensions(); - for (const auto &cell: _tensor.cells()) { + for (const auto &cell: _tensor.my_cells()) { addr.clear(); for (size_t i = 0; i < dimensions.size(); ++i) { if (dimensions[i].is_indexed()) { @@ -70,7 +70,7 @@ WrappedSimpleTensor::get_memory_usage() const Tensor::UP WrappedSimpleTensor::clone() const { - auto tensor = std::make_unique<eval::SimpleTensor>(_tensor.type(), _tensor.cells()); + auto tensor = std::make_unique<eval::SimpleTensor>(_tensor.type(), _tensor.my_cells()); return std::make_unique<WrappedSimpleTensor>(std::move(tensor)); } diff --git a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h index cf9103a4895..9c4031ac171 100644 --- a/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h +++ b/eval/src/vespa/eval/tensor/wrapped_simple_tensor.h @@ -26,6 +26,8 @@ public: : _space(), _tensor(tensor) {} explicit WrappedSimpleTensor(std::unique_ptr<eval::SimpleTensor> tensor) : _space(std::move(tensor)), _tensor(*_space) {} + TypedCells cells() const override { abort(); } + const Index &index() const override { abort(); } ~WrappedSimpleTensor() {} const eval::SimpleTensor &get() const { return _tensor; } const eval::ValueType &type() const override { return _tensor.type(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b704516bd2d..291011f91d9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -98,12 +98,6 @@ public class Flags { "Takes effect on next node agent tick. Change is orchestrated, but does NOT require container restart", HOSTNAME, APPLICATION_ID); - public static final UnboundStringFlag TLS_INSECURE_AUTHORIZATION_MODE = defineStringFlag( - "tls-insecure-authorization-mode", "log_only", - "TLS insecure authorization mode. Allowed values: ['disable', 'log_only', 'enforce']", - "Takes effect on restart of Docker container", - NODE_TYPE, APPLICATION_ID, HOSTNAME); - public static final UnboundIntFlag REBOOT_INTERVAL_IN_DAYS = defineIntFlag( "reboot-interval-in-days", 30, "No reboots are scheduled 0x-1x reboot intervals after the previous reboot, while reboot is " + @@ -157,21 +151,25 @@ public class Flags { "Selects type of sequenced executor used for feeding, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", "Selects type of sequenced executor used for mbus responses, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag RESPONSE_NUM_THREADS = defineIntFlag( "response-num-threads", 2, "Number of threads used for mbus responses, default is 2, negative number = numcores/4", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( "skip-communicatiomanager-thread", false, "Should we skip the communicationmanager thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, "Should we skip the mbus request thread", @@ -196,6 +194,24 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag TLS_COMPRESSION_TYPE = defineStringFlag( + "tls-compression-type", "NONE", + "Selects type of compression, valid values are NONE, NONE_MULTI, LZ4, ZSTD", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag TLS_USE_FSYNC = defineFeatureFlag( + "tls-use-fsync", false, + "Whether to use fsync when writing to the TLS.", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundDoubleFlag VISIBILITY_DELAY = defineDoubleFlag( + "visibility-delay", 0.0, + "Default visibility-delay", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_DIRECT_STORAGE_API_RPC = defineFeatureFlag( "use-direct-storage-api-rpc", false, "Whether to use direct RPC for Storage API communication between content cluster nodes.", @@ -270,7 +286,6 @@ public class Flags { "Whether to provision and use endpoint certs for apps in shared routing zones", "Takes effect on next deployment of the application", APPLICATION_ID); - public static final UnboundBooleanFlag USE_CLOUD_INIT_FORMAT = defineFeatureFlag( "use-cloud-init", false, "Use the cloud-init format when provisioning hosts", @@ -278,7 +293,7 @@ public class Flags { ZONE_ID); public static final UnboundBooleanFlag CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE = defineFeatureFlag( - "configserver-distribute-application-package", false, + "configserver-distribute-application-package", true, "Whether the application package should be distributed to other config servers during a deployment", "Takes effect immediately"); @@ -368,6 +383,39 @@ public class Flags { APPLICATION_ID ); + public static final UnboundBooleanFlag USE_CONFIG_SERVER_VIP = defineFeatureFlag( + "use-config-server-vip", + false, + "Whether the controller should use a config server VIP or not", + "Takes effect immediately", + ZONE_ID + ); + + public static final UnboundBooleanFlag SKIP_MAINTENANCE_DEPLOYMENT = defineFeatureFlag( + "node-repository-skip-maintenance-deployment", + false, + "Whether PeriodicApplicationMaintainer should skip deployment for an application", + "Takes effect at next run of maintainer", + APPLICATION_ID); + + public static final UnboundBooleanFlag DEPLOY_WITH_INTERNAL_RESTART = defineFeatureFlag( + "deploy-with-internal-restart", false, + "Whether controller should deploy application with internal restart parameter set", + "Takes effect on next deploy from controller", + APPLICATION_ID, ZONE_ID); + + public static final UnboundLongFlag NODE_OBJECT_CACHE_SIZE = defineLongFlag( + "node-object-cache-size", + 2000, + "The number of deserialized Node objects to store in-memory.", + "Takes effect on config server restart"); + + public static final UnboundBooleanFlag USE_NEW_RESTAPI_HANDLER = defineFeatureFlag( + "use-new-restapi-handler", + false, + "Whether application containers should use the new restapi handler implementation", + "Takes effect on next internal redeployment"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp index 3b37aa9a1bc..593bbd173c6 100644 --- a/fnet/src/vespa/fnet/frt/values.cpp +++ b/fnet/src/vespa/fnet/frt/values.cpp @@ -4,6 +4,7 @@ #include <vespa/fnet/databuffer.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/data/databuffer.h> #include <cassert> static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte."); @@ -300,6 +301,12 @@ FRT_Values::AddData(vespalib::alloc::Alloc && buf, uint32_t len) { } void +FRT_Values::AddData(vespalib::DataBuffer && buf) { + const auto len = buf.getDataLen(); + AddSharedData(&_stash.create<LocalBlob>(std::move(buf).stealBuffer(), len)); +} + +void FRT_Values::AddData(const char *buf, uint32_t len) { if (len > SHARED_LIMIT) { return AddSharedData(&_stash.create<LocalBlob>(buf, len)); diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h index 2aa7551c423..bac1f609fbb 100644 --- a/fnet/src/vespa/fnet/frt/values.h +++ b/fnet/src/vespa/fnet/frt/values.h @@ -5,7 +5,10 @@ #include "isharedblob.h" #include <cstring> -namespace vespalib { class Stash; } +namespace vespalib { + class Stash; + class DataBuffer; +} namespace vespalib::alloc { class Alloc; } namespace fnet { char * copyString(char *dst, const char *src, size_t len); @@ -218,6 +221,7 @@ public: FRT_StringValue *AddStringArray(uint32_t len); void AddSharedData(FRT_ISharedBlob *blob); void AddData(Alloc && buf, uint32_t len); + void AddData(vespalib::DataBuffer && buf); void AddData(const char *buf, uint32_t len); char *AddData(uint32_t len); FRT_DataValue *AddDataArray(uint32_t len); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java index c3d07a70e14..c3e386b020c 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java @@ -87,6 +87,7 @@ public class Response { int UNPROCESSABLE_ENTITY = 422; int LOCKED = 423; int FAILED_DEPENDENCY = 424; + int TOO_MANY_REQUESTS = 429; /** 5xx: Server Error - The server failed to fulfill an apparently valid request. */ int INTERNAL_SERVER_ERROR = 500; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java index 9ced933105d..fc9a6fc03be 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java @@ -67,8 +67,8 @@ class SslHandshakeFailedListener implements SslHandshakeListener { // Note: this pattern will match certificates with too late notBefore as well "PKIX path validation failed: java.security.cert.CertPathValidatorException: validity check failed"), INVALID_CLIENT_CERT( - Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, - "PKIX path (building|validation) failed: .+"); + Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, // Includes mismatch of client certificate and private key + "(PKIX path (building|validation) failed: .+)|(Invalid CertificateVerify signature)"); private final String metricName; private final Predicate<String> messageMatcher; diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json index 159c82971cd..4bec4bd91fe 100644 --- a/messagebus/abi-spec.json +++ b/messagebus/abi-spec.json @@ -362,7 +362,9 @@ "public int getMaxPendingCount()", "public com.yahoo.messagebus.MessageBusParams setMaxPendingCount(int)", "public int getMaxPendingSize()", - "public com.yahoo.messagebus.MessageBusParams setMaxPendingSize(int)" + "public com.yahoo.messagebus.MessageBusParams setMaxPendingSize(int)", + "public com.yahoo.messagebus.MessagebusConfig getMessageBusConfig()", + "public com.yahoo.messagebus.MessageBusParams setMessageBusConfig(com.yahoo.messagebus.MessagebusConfig)" ], "fields": [] }, @@ -643,6 +645,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.messagebus.MessageBusParams, com.yahoo.messagebus.network.rpc.RPCNetworkParams)", "public void <init>(com.yahoo.messagebus.MessageBusParams, com.yahoo.messagebus.network.rpc.RPCNetworkParams, java.lang.String)", "public void <init>(java.util.List, com.yahoo.messagebus.network.rpc.RPCNetworkParams, java.lang.String)", "public void <init>(com.yahoo.messagebus.Protocol, java.lang.String)", diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java index 7f55401cf43..2c8382ab715 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBusParams.java @@ -19,6 +19,7 @@ public class MessageBusParams { private RetryPolicy retryPolicy; private int maxPendingCount; private int maxPendingSize; + private MessagebusConfig config; /** * Constructs a new instance of this parameter object with default values for all members. @@ -27,6 +28,7 @@ public class MessageBusParams { retryPolicy = new RetryTransientErrorsPolicy(); maxPendingCount = 1024; maxPendingSize = 128 * 1024 * 1024; + config = null; } /** @@ -39,6 +41,7 @@ public class MessageBusParams { retryPolicy = params.retryPolicy; maxPendingCount = params.maxPendingCount; maxPendingSize = params.maxPendingSize; + config = params.config; } /** @@ -143,4 +146,14 @@ public class MessageBusParams { this.maxPendingSize = maxSize; return this; } + + public MessagebusConfig getMessageBusConfig() { + return config; + } + + public MessageBusParams setMessageBusConfig(MessagebusConfig config) { + this.config = config; + return this; + } + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java index 5ea278c410b..f5bf03c5420 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java @@ -21,6 +21,21 @@ public class RPCMessageBus extends NetworkMessageBus { /** * Constructs a new instance of this class. * + * @param mbusParams A complete set of message bus parameters, including messagebus config. + * @param rpcParams A complete set of network parameters, including rpc network config. + */ + public RPCMessageBus(MessageBusParams mbusParams, RPCNetworkParams rpcParams) { + this(mbusParams, new RPCNetwork(rpcParams)); + } + + private RPCMessageBus(MessageBusParams mbusParams, RPCNetwork network) { + super(network, new MessageBus(network, mbusParams)); + configAgent = new ConfigAgent(mbusParams.getMessageBusConfig(), getMessageBus()); + } + + /** + * Constructs a new instance of this class. + * * @param mbusParams A complete set of message bus parameters. * @param rpcParams A complete set of network parameters. * @param routingCfgId The config id for message bus routing specs. diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java index 032f13a008a..b5cbfd8224e 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingSpec.java @@ -195,7 +195,6 @@ public class RoutingSpec { return ret.toString(); } - // Overrides Object. @Override public String toString() { StringBuilder ret = new StringBuilder(); @@ -203,7 +202,6 @@ public class RoutingSpec { return ret.toString(); } - // Overrides Object. @Override public boolean equals(Object obj) { if (!(obj instanceof RoutingSpec)) { @@ -218,7 +216,7 @@ public class RoutingSpec { @Override public int hashCode() { - int result = tables != null ? tables.hashCode() : 0; + int result = tables.hashCode(); result = 31 * result + (verify ? 1 : 0); return result; } diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp index 6afa1528092..f7303ece20f 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp @@ -4,7 +4,6 @@ #include "rpcnetwork.h" #include "rpcserviceaddress.h" #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/tracelevel.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/databuffer.h> @@ -49,7 +48,8 @@ Memory SERVICE_F("service"); } -bool RPCSendV2::isCompatible(stringref method, stringref request, stringref response) +bool +RPCSendV2::isCompatible(stringref method, stringref request, stringref response) { return (method == METHOD_NAME) && (request == METHOD_PARAMS) && @@ -133,7 +133,7 @@ RPCSendV2::encodeRequest(FRT_RPCRequest &req, const Version &version, const Rout args.AddInt32(toCompress.size()); const auto bufferLength = buf.getDataLen(); assert(bufferLength <= INT32_MAX); - args.AddData(buf.stealBuffer(), bufferLength); + args.AddData(std::move(buf).stealBuffer(), bufferLength); } namespace { @@ -141,7 +141,7 @@ namespace { class ParamsV2 : public RPCSend::Params { public: - ParamsV2(const FRT_Values &arg) + explicit ParamsV2(const FRT_Values &arg) : _slime() { uint8_t encoding = arg[3]._intval8; @@ -261,9 +261,8 @@ RPCSendV2::createResponse(FRT_Values & ret, const string & version, Reply & repl ret.AddInt8(type); ret.AddInt32(toCompress.size()); - const auto bufferLength = buf.getDataLen(); - assert(bufferLength <= INT32_MAX); - ret.AddData(buf.stealBuffer(), bufferLength); + assert(buf.getDataLen() <= INT32_MAX); + ret.AddData(std::move(buf)); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java index 810596d6d0b..7c0302ae3dc 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java @@ -62,7 +62,7 @@ public class ConfigSentinelClientTest { } @Test - public void testElastic() throws Exception { + public void testElastic() { String response = "container state=RUNNING mode=AUTO pid=14338 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"get/container.0\"\n" + "container-clustercontroller state=RUNNING mode=AUTO pid=25020 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/cluster-controllers/0\"\n" + "distributor state=RUNNING mode=AUTO pid=25024 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/distributor/0\"\n" + @@ -72,7 +72,6 @@ public class ConfigSentinelClientTest { "metricsproxy state=RUNNING mode=AUTO pid=13107 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/metricsproxy\"\n" + "searchnode state=RUNNING mode=AUTO pid=25023 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/0\"\n" + "slobrok state=RUNNING mode=AUTO pid=25019 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" + - "topleveldispatch state=RUNNING mode=AUTO pid=25026 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/tlds/tld.0\"\n" + "\n"; ConfigSentinelDummy configsentinel = new ConfigSentinelDummy(response); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java index 5c746521c1f..def57617f56 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java @@ -15,8 +15,6 @@ public class ConfigSentinelDummy { + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" @@ -43,8 +41,6 @@ public class ConfigSentinelDummy { + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index b08dc6bbaf2..3b3711d1081 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -43,6 +43,7 @@ public final class Node { private final Reports reports; private final Optional<String> modelName; private final Optional<TenantName> reservedTo; + private final Optional<String> switchHostname; /** Record of the last event of each type happening to this node */ private final History history; @@ -54,20 +55,21 @@ public final class Node { public static Node createDockerNode(Set<String> ipAddresses, String hostname, String parentHostname, NodeResources resources, NodeType type) { return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname), new Flavor(resources), Status.initial(), State.reserved, - Optional.empty(), History.empty(), type, new Reports(), Optional.empty(), Optional.empty()); + Optional.empty(), History.empty(), type, new Reports(), Optional.empty(), Optional.empty(), + Optional.empty()); } /** Creates a node in the initial state (provisioned) */ public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, - Optional<String> modelName, Flavor flavor, Optional<TenantName> reservedTo, NodeType type) { + Optional<String> modelName, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, Optional<String> switchHostname) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, Status.initial(), State.provisioned, - Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo); + Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo, switchHostname); } /** Creates a node. See also the {@code create} helper methods. */ public Node(String id, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type, - Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo) { + Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo, Optional<String> switchHostname) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); this.ipConfig = Objects.requireNonNull(ipConfig, "A node must a have an IP config"); @@ -81,6 +83,7 @@ public final class Node { this.reports = Objects.requireNonNull(reports, "A null reports is not permitted"); this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted"); this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); + this.switchHostname = requireNonEmptyString(switchHostname, "switchHostname cannot be null"); if (state == State.active) requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); @@ -88,6 +91,7 @@ public final class Node { if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); + if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); } if (type != NodeType.host && reservedTo.isPresent()) @@ -151,7 +155,7 @@ public final class Node { /** Returns all the reports on this node. */ public Reports reports() { return reports; } - /** Returns the hardware model of this node */ + /** Returns the hardware model of this node, if any */ public Optional<String> modelName() { return modelName; } /** @@ -160,6 +164,11 @@ public final class Node { */ public Optional<TenantName> reservedTo() { return reservedTo; } + /** Returns the hostname of the switch this node is connected to, if any */ + public Optional<String> switchHostname() { + return switchHostname; + } + /** * Returns a copy of this node with wantToRetire and wantToDeprovision set to the given values and updated history. * @@ -217,42 +226,44 @@ public final class Node { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, + reports, modelName, reservedTo, switchHostname); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, + reports, modelName, reservedTo, switchHostname); } /** Returns a node with the flavor assigned to the given value */ public Node with(Flavor flavor) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, + reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this with the openStackId set */ public Node withOpenStackId(String openStackId) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.of(modelName), reservedTo); + allocation, history, type, reports, Optional.of(modelName), reservedTo, switchHostname); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.empty(), reservedTo); + allocation, history, type, reports, Optional.empty(), reservedTo, switchHostname); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -278,39 +289,50 @@ public final class Node { */ public Node with(Allocation allocation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.of(allocation), history, type, reports, modelName, reservedTo); + Optional.of(allocation), history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a new Node without an allocation. */ public Node withoutAllocation() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.empty(), history, type, reports, modelName, reservedTo); + Optional.empty(), history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } public Node withReservedTo(TenantName tenant) { if (type != NodeType.host) throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.of(tenant)); + allocation, history, type, reports, modelName, Optional.of(tenant), switchHostname); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.empty()); + allocation, history, type, reports, modelName, Optional.empty(), switchHostname); + } + + /** Returns a copy of this node with switch hostname set to given value */ + public Node withSwitchHostname(String switchHostname) { + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(switchHostname)); + } + + /** Returns a copy of this node with switch hostname unset */ + public Node withoutSwitchHostname() { + return withSwitchHostname(null); } /** Returns a copy of this node with the current reboot generation set to the given number at the given instant */ @@ -343,12 +365,12 @@ public final class Node { /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } public Node with(Reports reports) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo); + allocation, history, type, reports, modelName, reservedTo, switchHostname); } private static Optional<String> requireNonEmptyString(Optional<String> value, String message) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 4ec7ddd04c4..e70793111ea 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -155,7 +155,8 @@ public class NodeRepository extends AbstractComponent { int spareCount) { // Flag is read once here as it shouldn't not change at runtime this.useConfigServerLock = Flags.USE_CONFIG_SERVER_LOCK.bindTo(flagSource).value(); - this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, useConfigServerLock); + long nodeObjectCacheSize = Flags.NODE_OBJECT_CACHE_SIZE.bindTo(flagSource).value(); + this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, useConfigServerLock, nodeObjectCacheSize); this.zone = zone; this.clock = clock; this.flavors = flavors; @@ -407,7 +408,7 @@ public class NodeRepository extends AbstractComponent { Flavor flavor, Optional<TenantName> reservedTo, NodeType type) { if (ipConfig.primary().isEmpty()) // TODO: Remove this. Only test code hits this path ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname)); - return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type); + return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type, Optional.empty()); } public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 5df45bbc1b1..b41b7f15499 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -79,9 +79,9 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { * @return whether it was successfully deployed */ protected final boolean deployWithLock(ApplicationId application) { + if ( ! canDeployNow(application)) return false; // redeployment is no longer needed try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { if ( ! deployment.isValid()) return false; // this will be done at another config server - if ( ! canDeployNow(application)) return false; // redeployment is no longer needed log.log(Level.INFO, application + " will be deployed, last deploy time " + getLastDeployTime(application)); return deployment.activate().isPresent(); } finally { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index e0d7dc5f19e..cd05c7e36d3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.persistence.CacheStats; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -68,9 +69,22 @@ public class MetricsReporter extends NodeRepositoryMaintainer { updateMaintenanceMetrics(); updateDockerMetrics(nodes); updateTenantUsageMetrics(nodes); + updateCacheMetrics(); return true; } + private void updateCacheMetrics() { + CacheStats nodeCacheStats = nodeRepository().database().nodeSerializerCacheStats(); + metric.set("cache.nodeObject.hitRate", nodeCacheStats.hitRate(), null); + metric.set("cache.nodeObject.evictionCount", nodeCacheStats.evictionCount(), null); + metric.set("cache.nodeObject.size", nodeCacheStats.size(), null); + + CacheStats curatorCacheStats = nodeRepository().database().cacheStats(); + metric.set("cache.curator.hitRate", curatorCacheStats.hitRate(), null); + metric.set("cache.curator.evictionCount", curatorCacheStats.evictionCount(), null); + metric.set("cache.curator.size", curatorCacheStats.size(), null); + } + private void updateMaintenanceMetrics() { metric.set("hostedVespa.pendingRedeployments", pendingRedeploymentsSupplier.get(), null); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index a2a189769bf..77ef88f0952 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -75,10 +75,10 @@ public class NodeFailer extends NodeRepositoryMaintainer { public NodeFailer(Deployer deployer, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, NodeRepository nodeRepository, - Duration downTimeLimit, Clock clock, Orchestrator orchestrator, + Duration downTimeLimit, Duration interval, Clock clock, Orchestrator orchestrator, ThrottlePolicy throttlePolicy, Metric metric) { - // check ping status every five minutes, but at least twice as often as the down time limit - super(nodeRepository, min(downTimeLimit.dividedBy(2), Duration.ofMinutes(5)), metric); + // check ping status every interval, but at least twice as often as the down time limit + super(nodeRepository, min(downTimeLimit.dividedBy(2), interval), metric); this.deployer = deployer; this.hostLivenessTracker = hostLivenessTracker; this.serviceMonitor = serviceMonitor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 0890908dc80..0dd7cfe47f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; @@ -70,10 +71,12 @@ public class NodeRepositoryMaintenance extends AbstractComponent { Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) { - DefaultTimes defaults = new DefaultTimes(zone); + DefaultTimes defaults = new DefaultTimes(zone, Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource).value()); - nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); - periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval); + nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, + defaults.nodeFailerInterval, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); + periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, + defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval, flagSource); operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.operatorChangeRedeployInterval); reservationExpirer = new ReservationExpirer(nodeRepository, clock, defaults.reservationExpiry, metric); retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, clock, defaults.retiredInterval, defaults.retiredExpiry); @@ -155,6 +158,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration provisionedExpiry; private final Duration spareCapacityMaintenanceInterval; private final Duration metricsInterval; + private final Duration nodeFailerInterval; private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; private final Duration loadBalancerExpirerInterval; @@ -167,20 +171,21 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; - DefaultTimes(Zone zone) { - autoscalingInterval = Duration.ofMinutes(5); + DefaultTimes(Zone zone, boolean deploymentExistsOnAllConfigServers) { + autoscalingInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(15) : Duration.ofMinutes(5); dynamicProvisionerInterval = Duration.ofMinutes(5); failedExpirerInterval = Duration.ofMinutes(10); failGrace = Duration.ofMinutes(30); infrastructureProvisionInterval = Duration.ofMinutes(1); loadBalancerExpirerInterval = Duration.ofMinutes(5); metricsInterval = Duration.ofMinutes(1); + nodeFailerInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(15) : Duration.ofMinutes(5); nodeMetricsCollectionInterval = Duration.ofMinutes(1); - operatorChangeRedeployInterval = Duration.ofMinutes(1); + operatorChangeRedeployInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(3) : Duration.ofMinutes(1); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); periodicRedeployInterval = Duration.ofMinutes(30); provisionedExpiry = Duration.ofHours(4); - rebalancerInterval = Duration.ofMinutes(40); + rebalancerInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(120) : Duration.ofMinutes(40); redeployMaintainerInterval = Duration.ofMinutes(1); // Need to be long enough for deployment to be finished for all config model versions // Should be equal to timeout for deployments @@ -191,7 +196,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy - retiredInterval = Duration.ofMinutes(10); + retiredInterval = deploymentExistsOnAllConfigServers ? Duration.ofMinutes(30) : Duration.ofMinutes(10); dirtyExpiry = Duration.ofHours(2); // enough time to clean the node retiredExpiry = Duration.ofDays(4); // give up migrating data after 4 days } else { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index e3128dfc8e9..289d5a6742a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -4,6 +4,10 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -26,11 +30,13 @@ import java.util.stream.Collectors; public class PeriodicApplicationMaintainer extends ApplicationMaintainer { private final Duration minTimeBetweenRedeployments; + private final FlagSource flagSource; PeriodicApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository, - Duration interval, Duration minTimeBetweenRedeployments) { + Duration interval, Duration minTimeBetweenRedeployments, FlagSource flagSource) { super(deployer, metric, nodeRepository, interval); this.minTimeBetweenRedeployments = minTimeBetweenRedeployments; + this.flagSource = flagSource; } @Override @@ -64,10 +70,9 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { } private boolean shouldMaintain(ApplicationId id) { - if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking")) return false; - if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking-canary")) return false; - if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking-rhel7")) return false; - return true; + BooleanFlag skipMaintenanceDeployment = Flags.SKIP_MAINTENANCE_DEPLOYMENT.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, id.serializedForm()); + return ! skipMaintenanceDeployment.value(); } protected List<Node> nodesNeedingMaintenance() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index b0c52d10f7d..db0e5f03097 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -62,7 +62,8 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { Optional<Cluster> cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; Optional<ClusterResources> suggestion = autoscaler.suggest(cluster.get(), clusterNodes); - try (Mutex lock = nodeRepository().lock(applicationId)) { + // Wait only a short time for the lock to avoid interfering with change deployments + try (Mutex lock = nodeRepository().lock(applicationId, Duration.ofSeconds(1))) { applications().get(applicationId).ifPresent(a -> storeSuggestion(suggestion, clusterId, a, lock)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 836583022ca..66061ad4e9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -9,8 +9,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; -import java.net.Inet4Address; -import java.net.Inet6Address; +import java.net.InetAddress; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -33,9 +32,9 @@ import static com.yahoo.config.provision.NodeType.proxyhost; public class IP { /** Comparator for sorting IP addresses by their natural order */ - public static final Comparator<String> NATURAL_ORDER = (ip1, ip2) -> { - byte[] address1 = InetAddresses.forString(ip1).getAddress(); - byte[] address2 = InetAddresses.forString(ip2).getAddress(); + public static final Comparator<InetAddress> NATURAL_ORDER = (ip1, ip2) -> { + byte[] address1 = ip1.getAddress(); + byte[] address2 = ip2.getAddress(); // IPv4 always sorts before IPv6 if (address1.length < address2.length) return -1; @@ -88,7 +87,7 @@ public class IP { /** Returns a copy of this with pool set to given value */ public Config with(Set<String> primary) { - return new Config(require(primary), pool.asSet()); + return new Config(primary, pool.asSet()); } @Override @@ -110,16 +109,6 @@ public class IP { return String.format("ip config primary=%s pool=%s", primary, pool.asSet()); } - /** Validates and returns the given addresses */ - public static Set<String> require(Set<String> addresses) { - try { - addresses.forEach(InetAddresses::forString); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Found one or more invalid addresses in " + addresses, e); - } - return addresses; - } - /** * Verify IP config of given nodes * @@ -414,14 +403,28 @@ public class IP { } + /** Validate IP address*/ + public static InetAddress parse(String ipAddress) { + try { + return InetAddresses.forString(ipAddress); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid IP address '" + ipAddress + "'", e); + } + } + + /** Convert IP address to string. This uses :: for zero compression in IPv6 addresses. */ + public static String asString(InetAddress inetAddress) { + return InetAddresses.toAddrString(inetAddress); + } + /** Returns whether given string is an IPv4 address */ public static boolean isV4(String ipAddress) { - return InetAddresses.forString(ipAddress) instanceof Inet4Address; + return ipAddress.contains("."); } /** Returns whether given string is an IPv6 address */ public static boolean isV6(String ipAddress) { - return InetAddresses.forString(ipAddress) instanceof Inet6Address; + return ipAddress.contains(":"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java new file mode 100644 index 00000000000..a2a2bcf3ab4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CacheStats.java @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +/** + * Statistics for caches used by {@link com.yahoo.vespa.hosted.provision.NodeRepository}. + * + * @author mpolden + */ +public class CacheStats { + + private final double hitRate; + private final long evictionCount; + private final long size; + + public CacheStats(double hitRate, long evictionCount, long size) { + this.hitRate = hitRate; + this.evictionCount = evictionCount; + this.size = size; + } + + /** The fraction of lookups that resulted in a hit */ + public double hitRate() { + return hitRate; + } + + /** The number of entries that have been evicted */ + public long evictionCount() { + return evictionCount; + } + + /** The current size of the cache */ + public long size() { + return size; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java index 08f2cfec40f..fa5a72eea52 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.google.common.cache.AbstractCache; import com.google.common.collect.ImmutableList; import com.yahoo.config.provision.HostName; import com.yahoo.path.Path; @@ -17,6 +18,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -116,6 +118,10 @@ public class CuratorDatabase { return cache.get(); } + CacheStats cacheStats() { + return cache.get().stats(); + } + /** Caches must only be instantiated using this method */ private Cache newCache(long generation) { return useCache ? new Cache(generation, curator) : new NoCache(generation, curator); @@ -139,6 +145,8 @@ public class CuratorDatabase { private final Map<Path, List<String>> children = new ConcurrentHashMap<>(); private final Map<Path, Optional<byte[]>> data = new ConcurrentHashMap<>(); + private final AbstractCache.SimpleStatsCounter stats = new AbstractCache.SimpleStatsCounter(); + /** Create an empty snapshot at a given generation (as an empty snapshot is a valid partial snapshot) */ private Cache(long generation, Curator curator) { this.generation = generation; @@ -146,13 +154,29 @@ public class CuratorDatabase { } @Override - public List<String> getChildren(Path path) { - return children.computeIfAbsent(path, key -> ImmutableList.copyOf(curator.getChildren(path))); + public List<String> getChildren(Path path) { + return get(children, path, () -> ImmutableList.copyOf(curator.getChildren(path))); } @Override public Optional<byte[]> getData(Path path) { - return data.computeIfAbsent(path, key -> curator.getData(path)).map(data -> Arrays.copyOf(data, data.length)); + return get(data, path, () -> curator.getData(path)).map(data -> Arrays.copyOf(data, data.length)); + } + + private <T> T get(Map<Path, T> values, Path path, Supplier<T> loader) { + return values.compute(path, (key, value) -> { + if (value == null) { + stats.recordMisses(1); + return loader.get(); + } + stats.recordHits(1); + return value; + }); + } + + public CacheStats stats() { + var stats = this.stats.snapshot(); + return new CacheStats(stats.hitRate(), stats.evictionCount(), children.size() + data.size()); } } @@ -183,4 +207,5 @@ public class CuratorDatabase { Optional<byte[]> getData(Path path); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index cc62ae67e84..56f7c951025 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -79,8 +79,9 @@ public class CuratorDatabaseClient { private final CuratorCounter provisionIndexCounter; private final boolean logStackTracesOnLockTimeout; - public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache, boolean logStackTracesOnLockTimeout) { - this.nodeSerializer = new NodeSerializer(flavors); + public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache, boolean logStackTracesOnLockTimeout, + long nodeObjectCacheSize) { + this.nodeSerializer = new NodeSerializer(flavors, nodeObjectCacheSize); this.zone = zone; this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; @@ -216,7 +217,8 @@ public class CuratorDatabaseClient { toState, toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), - node.type(), node.reports(), node.modelName(), node.reservedTo()); + node.type(), node.reports(), node.modelName(), node.reservedTo(), + node.switchHostname()); writeNode(toState, curatorTransaction, node, newNode); writtenNodes.add(newNode); } @@ -289,7 +291,7 @@ public class CuratorDatabaseClient { } /** - * Returns a particular node, or empty if this noe is not in any of the given states. + * Returns a particular node, or empty if this node is not in any of the given states. * If no states are given this returns the node if it is present in any state. */ public Optional<Node> readNode(CuratorDatabase.Session session, String hostname, Node.State ... states) { @@ -587,6 +589,14 @@ public class CuratorDatabaseClient { .collect(Collectors.toList()); } + public CacheStats cacheStats() { + return db.cacheStats(); + } + + public CacheStats nodeSerializerCacheStats() { + return nodeSerializer.cacheStats(); + } + private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { return db.getData(path).filter(data -> data.length > 0).map(mapper); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 5a3584b6ff4..397182e204e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -1,7 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hashing; +import com.google.common.util.concurrent.UncheckedExecutionException; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -32,12 +36,13 @@ import com.yahoo.vespa.hosted.provision.node.Reports; import com.yahoo.vespa.hosted.provision.node.Status; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.function.UnaryOperator; +import java.util.concurrent.ExecutionException; /** * Serializes a node to/from JSON. @@ -79,6 +84,7 @@ public class NodeSerializer { private static final String reportsKey = "reports"; private static final String modelNameKey = "modelName"; private static final String reservedToKey = "reservedTo"; + private static final String switchHostnameKey = "switchHostname"; // Node resource fields private static final String flavorKey = "flavor"; @@ -106,10 +112,17 @@ public class NodeSerializer { // Network port fields private static final String networkPortsKey = "networkPorts"; + // A cache of deserialized Node objects. The cache is keyed on the hash of serialized node data. + // + // Deserializing a Node from slime is expensive, and happens frequently. Node instances that have already been + // deserialized are returned from this cache instead of being deserialized again. + private final Cache<Long, Node> cache; + // ---------------- Serialization ---------------------------------------------------- - public NodeSerializer(NodeFlavors flavors) { + public NodeSerializer(NodeFlavors flavors, long cacheSize) { this.flavors = flavors; + this.cache = CacheBuilder.newBuilder().maximumSize(cacheSize).recordStats().build(); } public byte[] toJson(Node node) { @@ -123,10 +136,16 @@ public class NodeSerializer { } } + /** Returns cache statistics for this serializer */ + public CacheStats cacheStats() { + var stats = cache.stats(); + return new CacheStats(stats.hitRate(), stats.evictionCount(), cache.size()); + } + private void toSlime(Node node, Cursor object) { object.setString(hostnameKey, node.hostname()); - toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey), IP.Config::require); - toSlime(node.ipConfig().pool().asSet(), object.setArray(ipAddressPoolKey), UnaryOperator.identity() /* Pool already holds a validated address list */); + toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey)); + toSlime(node.ipConfig().pool().asSet(), object.setArray(ipAddressPoolKey)); object.setString(idKey, node.id()); node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname)); toSlime(node.flavor(), object); @@ -143,6 +162,7 @@ public class NodeSerializer { node.status().osVersion().current().ifPresent(version -> object.setString(osVersionKey, version.toString())); node.status().osVersion().wanted().ifPresent(version -> object.setString(wantedOsVersionKey, version.toFullString())); node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong(firmwareCheckKey, instant.toEpochMilli())); + node.switchHostname().ifPresent(switchHostname -> object.setString(switchHostnameKey, switchHostname)); node.reports().toSlime(object, reportsKey); node.modelName().ifPresent(modelName -> object.setString(modelNameKey, modelName)); node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value())); @@ -186,15 +206,24 @@ public class NodeSerializer { object.setString(agentKey, toString(event.agent())); } - private void toSlime(Set<String> ipAddresses, Cursor array, UnaryOperator<Set<String>> validator) { - // Sorting IP addresses is expensive, so we do it at serialization time instead of Node construction time - validator.apply(ipAddresses).stream().sorted(IP.NATURAL_ORDER).forEach(array::addString); + private void toSlime(Set<String> ipAddresses, Cursor array) { + // Validating IP address string literals is expensive, so we do it at serialization time instead of Node + // construction time + ipAddresses.stream().map(IP::parse).sorted(IP.NATURAL_ORDER).map(IP::asString).forEach(array::addString); } // ---------------- Deserialization -------------------------------------------------- public Node fromJson(Node.State state, byte[] data) { - return nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get()); + var key = Hashing.sipHash24().newHasher() + .putString(state.name(), StandardCharsets.UTF_8) + .putBytes(data).hash() + .asLong(); + try { + return cache.get(key, () -> nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get())); + } catch (ExecutionException e) { + throw new UncheckedExecutionException(e); + } } private Node nodeFromSlime(Node.State state, Inspector object) { @@ -212,7 +241,8 @@ public class NodeSerializer { nodeTypeFromString(object.field(nodeTypeKey).asString()), Reports.fromSlime(object.field(reportsKey)), modelNameFromSlime(object), - reservedToFromSlime(object.field(reservedToKey))); + reservedToFromSlime(object.field(reservedToKey)), + switchHostnameFromSlime(object.field(switchHostnameKey))); } private Status statusFromSlime(Inspector object) { @@ -227,6 +257,11 @@ public class NodeSerializer { instantFromSlime(object.field(firmwareCheckKey))); } + private Optional<String> switchHostnameFromSlime(Inspector field) { + if (!field.valid()) return Optional.empty(); + return Optional.of(field.asString()); + } + private Flavor flavorFromSlime(Inspector object) { Inspector resources = object.field(resourcesKey); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index d3faa4d80f5..75f3c892571 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -99,13 +99,13 @@ public class GroupPreparer { nodeRepository.addNodes(hosts, Agent.application); // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit - List<PrioritizableNode> nodes = provisionedHosts.stream() - .map(provisionedHost -> new PrioritizableNode.Builder(provisionedHost.generateNode()) - .parent(provisionedHost.generateHost()) - .newNode(true) - .build()) - .collect(Collectors.toList()); - allocation.offer(nodes); + List<NodeCandidate> candidates = provisionedHosts.stream() + .map(provisionedHost -> new NodeCandidate.Builder(provisionedHost.generateNode()) + .parent(provisionedHost.generateHost()) + .newNode(true) + .build()) + .collect(Collectors.toList()); + allocation.offer(candidates); } if (! allocation.fulfilled() && requestedNodes.canFail()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index a37da10f5f0..b07ce786685 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -21,8 +21,9 @@ import java.util.Collection; import java.util.Comparator; import java.util.EnumSet; import java.util.HashSet; -import java.util.LinkedHashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -48,8 +49,8 @@ class NodeAllocation { /** The requested nodes of this list */ private final NodeSpec requestedNodes; - /** The nodes this has accepted so far */ - private final Set<PrioritizableNode> nodes = new LinkedHashSet<>(); + /** The node candidates this has accepted so far, keyed on hostname */ + private final Map<String, NodeCandidate> nodes = new LinkedHashMap<>(); /** The number of already allocated nodes accepted and not retired */ private int accepted = 0; @@ -98,36 +99,36 @@ class NodeAllocation { * @param nodesPrioritized the nodes which are potentially on offer. These may belong to a different application etc. * @return the subset of offeredNodes which was accepted, with the correct allocation assigned */ - List<Node> offer(List<PrioritizableNode> nodesPrioritized) { + List<Node> offer(List<NodeCandidate> nodesPrioritized) { List<Node> accepted = new ArrayList<>(); - for (PrioritizableNode node : nodesPrioritized) { - Node offered = node.node; + for (NodeCandidate candidate : nodesPrioritized) { + Node offered = candidate.node; if (offered.allocation().isPresent()) { Allocation allocation = offered.allocation().get(); ClusterMembership membership = allocation.membership(); if ( ! allocation.owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type - if ((! node.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it + if ((! candidate.isSurplusNode || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group and we can't or have no reason to change it if ( offered.state() == Node.State.active && allocation.isRemovable()) continue; // don't accept; causes removal if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) boolean resizeable = false; boolean acceptToRetire = false; if (requestedNodes.considerRetiring()) { - resizeable = node.isResizable; - acceptToRetire = acceptToRetire(node); + resizeable = candidate.isResizable; + acceptToRetire = acceptToRetire(candidate); } - if ((! saturated() && hasCompatibleFlavor(node) && requestedNodes.acceptable(offered)) || acceptToRetire) - accepted.add(acceptNode(node, shouldRetire(node), resizeable)); + if ((! saturated() && hasCompatibleFlavor(candidate) && requestedNodes.acceptable(offered)) || acceptToRetire) + accepted.add(acceptNode(candidate, shouldRetire(candidate), resizeable)); } - else if (! saturated() && hasCompatibleFlavor(node)) { + else if (! saturated() && hasCompatibleFlavor(candidate)) { if ( ! nodeResourceLimits.isWithinRealLimits(offered, cluster)) { ++rejectedDueToInsufficientRealResources; continue; } - if ( violatesParentHostPolicy(this.nodes, offered)) { + if ( violatesParentHostPolicy(offered)) { ++rejectedDueToClashingParentHost; continue; } @@ -142,29 +143,29 @@ class NodeAllocation { if (offered.status().wantToRetire()) { continue; } - node.node = offered.allocate(application, - ClusterMembership.from(cluster, highestIndex.add(1)), - requestedNodes.resources().orElse(node.node.resources()), - nodeRepository.clock().instant()); - accepted.add(acceptNode(node, false, false)); + candidate = candidate.withNode(offered.allocate(application, + ClusterMembership.from(cluster, highestIndex.add(1)), + requestedNodes.resources().orElse(candidate.node.resources()), + nodeRepository.clock().instant())); + accepted.add(acceptNode(candidate, false, false)); } } return accepted; } - private boolean shouldRetire(PrioritizableNode node) { + private boolean shouldRetire(NodeCandidate candidate) { if ( ! requestedNodes.considerRetiring()) return false; - if ( ! nodeResourceLimits.isWithinRealLimits(node.node, cluster)) return true; - if (violatesParentHostPolicy(this.nodes, node.node)) return true; - if ( ! hasCompatibleFlavor(node)) return true; - if (node.node.status().wantToRetire()) return true; - if (requestedNodes.isExclusive() && ! hostsOnly(application, node.node.parentHostname())) return true; + if ( ! nodeResourceLimits.isWithinRealLimits(candidate.node, cluster)) return true; + if (violatesParentHostPolicy(candidate.node)) return true; + if ( ! hasCompatibleFlavor(candidate)) return true; + if (candidate.node.status().wantToRetire()) return true; + if (requestedNodes.isExclusive() && ! hostsOnly(application, candidate.node.parentHostname())) return true; return false; } - private boolean violatesParentHostPolicy(Collection<PrioritizableNode> accepted, Node offered) { - return checkForClashingParentHost() && offeredNodeHasParentHostnameAlreadyAccepted(accepted, offered); + private boolean violatesParentHostPolicy(Node offered) { + return checkForClashingParentHost() && offeredNodeHasParentHostnameAlreadyAccepted(offered); } private boolean checkForClashingParentHost() { @@ -173,8 +174,8 @@ class NodeAllocation { ! application.instance().isTester(); } - private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<PrioritizableNode> accepted, Node offered) { - for (PrioritizableNode acceptedNode : accepted) { + private boolean offeredNodeHasParentHostnameAlreadyAccepted(Node offered) { + for (NodeCandidate acceptedNode : nodes.values()) { if (acceptedNode.node.parentHostname().isPresent() && offered.parentHostname().isPresent() && acceptedNode.node.parentHostname().get().equals(offered.parentHostname().get())) { return true; @@ -230,21 +231,21 @@ class NodeAllocation { * initialized. (In the other case, where a container node is not desired because we have enough nodes we * do want to remove it immediately to get immediate feedback on how the size reduction works out.) */ - private boolean acceptToRetire(PrioritizableNode node) { - if (node.node.state() != Node.State.active) return false; - if (! node.node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; - if (node.node.allocation().get().membership().retired()) return true; // don't second-guess if already retired + private boolean acceptToRetire(NodeCandidate candidate) { + if (candidate.node.state() != Node.State.active) return false; + if (! candidate.node.allocation().get().membership().cluster().group().equals(cluster.group())) return false; + if (candidate.node.allocation().get().membership().retired()) return true; // don't second-guess if already retired return cluster.type().isContent() || - (cluster.type() == ClusterSpec.Type.container && !hasCompatibleFlavor(node)); + (cluster.type() == ClusterSpec.Type.container && !hasCompatibleFlavor(candidate)); } - private boolean hasCompatibleFlavor(PrioritizableNode node) { - return requestedNodes.isCompatible(node.node.flavor(), nodeRepository.flavors()) || node.isResizable; + private boolean hasCompatibleFlavor(NodeCandidate candidate) { + return requestedNodes.isCompatible(candidate.node.flavor(), nodeRepository.flavors()) || candidate.isResizable; } - private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resizeable) { - Node node = prioritizableNode.node; + private Node acceptNode(NodeCandidate candidate, boolean wantToRetire, boolean resizeable) { + Node node = candidate.node; if (node.allocation().isPresent()) // Record the currently requested resources node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.resources()))); @@ -271,13 +272,17 @@ class NodeAllocation { // group may be different node = setCluster(cluster, node); } - prioritizableNode.node = node; + candidate = candidate.withNode(node); indexes.add(node.allocation().get().membership().index()); highestIndex.set(Math.max(highestIndex.get(), node.allocation().get().membership().index())); - nodes.add(prioritizableNode); + put(candidate); return node; } + private void put(NodeCandidate candidate) { + nodes.put(candidate.node.hostname(), candidate); + } + private Node resize(Node node) { NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources(); return node.with(new Flavor(requestedNodes.resources().get() @@ -323,36 +328,39 @@ class NodeAllocation { * @return the final list of nodes */ List<Node> finalNodes() { - int currentRetiredCount = (int) nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); + int currentRetiredCount = (int) nodes.values().stream().filter(node -> node.node.allocation().get().membership().retired()).count(); int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), currentRetiredCount) - currentRetiredCount; if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0 - for (PrioritizableNode node : byRetiringPriority(nodes)) { - if ( ! node.node.allocation().get().membership().retired() && node.node.state() == Node.State.active) { - node.node = node.node.retire(Agent.application, nodeRepository.clock().instant()); + for (NodeCandidate candidate : byRetiringPriority(nodes.values())) { + if ( ! candidate.node.allocation().get().membership().retired() && candidate.node.state() == Node.State.active) { + candidate = candidate.withNode(candidate.node.retire(Agent.application, nodeRepository.clock().instant())); + put(candidate); if (--deltaRetiredCount == 0) break; } } } else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0 - for (PrioritizableNode node : byUnretiringPriority(nodes)) { - if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node) ) { - if (node.isResizable) - node.node = resize(node.node); - node.node = node.node.unretire(); + for (NodeCandidate candidate : byUnretiringPriority(nodes.values())) { + if ( candidate.node.allocation().get().membership().retired() && hasCompatibleFlavor(candidate) ) { + if (candidate.isResizable) + candidate = candidate.withNode(resize(candidate.node)); + candidate = candidate.withNode(candidate.node.unretire()); + put(candidate); if (++deltaRetiredCount == 0) break; } } } - for (PrioritizableNode node : nodes) { + for (NodeCandidate candidate : nodes.values()) { // Set whether the node is exclusive - Allocation allocation = node.node.allocation().get(); - node.node = node.node.with(allocation.with(allocation.membership() - .with(allocation.membership().cluster().exclusive(requestedNodes.isExclusive())))); + Allocation allocation = candidate.node.allocation().get(); + candidate = candidate.withNode(candidate.node.with(allocation.with(allocation.membership() + .with(allocation.membership().cluster().exclusive(requestedNodes.isExclusive()))))); + put(candidate); } - return nodes.stream().map(n -> n.node).collect(Collectors.toList()); + return nodes.values().stream().map(n -> n.node).collect(Collectors.toList()); } List<Node> reservableNodes() { @@ -361,32 +369,28 @@ class NodeAllocation { return nodesFilter(n -> !n.isNewNode && reservableStates.contains(n.node.state())); } - List<Node> surplusNodes() { - return nodesFilter(n -> n.isSurplusNode); - } - List<Node> newNodes() { return nodesFilter(n -> n.isNewNode); } - private List<Node> nodesFilter(Predicate<PrioritizableNode> predicate) { - return nodes.stream() + private List<Node> nodesFilter(Predicate<NodeCandidate> predicate) { + return nodes.values().stream() .filter(predicate) .map(n -> n.node) .collect(Collectors.toList()); } /** Prefer to retire nodes we want the least */ - private List<PrioritizableNode> byRetiringPriority(Set<PrioritizableNode> nodes) { - return nodes.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList()); + private List<NodeCandidate> byRetiringPriority(Collection<NodeCandidate> candidates) { + return candidates.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList()); } /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */ - private List<PrioritizableNode> byUnretiringPriority(Set<PrioritizableNode> nodes) { - return nodes.stream() - .sorted(Comparator.comparing((PrioritizableNode n) -> n.node.status().wantToRetire()) - .thenComparing(n -> n.node.allocation().get().membership().index())) - .collect(Collectors.toList()); + private List<NodeCandidate> byUnretiringPriority(Collection<NodeCandidate> candidates) { + return candidates.stream() + .sorted(Comparator.comparing((NodeCandidate n) -> n.node.status().wantToRetire()) + .thenComparing(n -> n.node.allocation().get().membership().index())) + .collect(Collectors.toList()); } public String outOfCapacityDetails() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 9955d75a742..651ab9b1e09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -8,11 +8,11 @@ import java.util.List; import java.util.Optional; /** - * A node with additional information required to prioritize it for allocation. + * A node candidate containing the details required to prioritize it for allocation. This is immutable. * * @author smorgrav */ -class PrioritizableNode implements Comparable<PrioritizableNode> { +class NodeCandidate implements Comparable<NodeCandidate> { /** List of host states ordered by preference (ascending) */ private static final List<Node.State> HOST_STATE_PRIORITY = @@ -21,8 +21,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { private static final NodeResources zeroResources = new NodeResources(0, 0, 0, 0, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); - // TODO: Make immutable - Node node; + final Node node; /** The free capacity on the parent of this node, before adding this node to it */ private final NodeResources freeParentCapacity; @@ -42,7 +41,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { /** This node can be resized to the new NodeResources */ final boolean isResizable; - PrioritizableNode(Node node, NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean isSurplusNode, boolean isNewNode, boolean isResizeable) { + NodeCandidate(Node node, NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean isSurplusNode, boolean isNewNode, boolean isResizeable) { if (isResizeable && isNewNode) throw new IllegalArgumentException("A new node cannot be resizable"); @@ -56,12 +55,12 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { } /** - * Compares two prioritizable nodes + * Compare this candidate to another * * @return negative if first priority is higher than second node */ @Override - public int compareTo(PrioritizableNode other) { + public int compareTo(NodeCandidate other) { // First always pick nodes without violation above nodes with violations if (!this.violatesSpares && other.violatesSpares) return -1; if (!other.violatesSpares && this.violatesSpares) return 1; @@ -138,7 +137,12 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { /** Returns the allocation skew of the parent of this after adding this node to it */ double skewWithThis() { return skewWith(node.resources()); } - private boolean lessThanHalfTheHost(PrioritizableNode node) { + /** Returns a copy of this with node set to given value */ + NodeCandidate withNode(Node node) { + return new NodeCandidate(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); + } + + private boolean lessThanHalfTheHost(NodeCandidate node) { var n = node.node.resources(); var h = node.parent.get().resources(); if (h.vcpu() < n.vcpu() * 2) return false; @@ -172,8 +176,8 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { @Override public boolean equals(Object other) { if (other == this) return true; - if ( ! (other instanceof PrioritizableNode)) return false; - return this.node.equals(((PrioritizableNode)other).node); + if ( ! (other instanceof NodeCandidate)) return false; + return this.node.equals(((NodeCandidate)other).node); } static class Builder { @@ -222,8 +226,8 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { return this; } - PrioritizableNode build() { - return new PrioritizableNode(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); + NodeCandidate build() { + return new NodeCandidate(node, freeParentCapacity, parent, violatesSpares, isSurplusNode, isNewNode, isResizable); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 3dc7eefa277..226079e43fe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -5,29 +5,28 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; - -import java.util.ArrayList; -import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; +import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; /** * Builds up data structures necessary for node prioritization. It wraps each node - * up in a PrioritizableNode object with attributes used in sorting. + * up in a {@link NodeCandidate} object with attributes used in sorting. * - * The actual sorting/prioritization is implemented in the PrioritizableNode class as a compare method. + * The prioritization logic is implemented by {@link NodeCandidate}. * * @author smorgrav */ @@ -35,7 +34,7 @@ public class NodePrioritizer { private final static Logger log = Logger.getLogger(NodePrioritizer.class.getName()); - private final Map<Node, PrioritizableNode> nodes = new HashMap<>(); + private final Map<Node, NodeCandidate> nodes = new HashMap<>(); private final LockedNodeList allNodes; private final HostCapacity capacity; private final NodeSpec requestedNodes; @@ -81,8 +80,8 @@ public class NodePrioritizer { this.isDocker = resources(requestedNodes) != null; } - /** Returns the list of nodes sorted by PrioritizableNode::compare */ - List<PrioritizableNode> prioritize() { + /** Returns the list of nodes sorted by {@link NodeCandidate#compareTo(NodeCandidate)} */ + List<NodeCandidate> prioritize() { return nodes.values().stream().sorted().collect(Collectors.toList()); } @@ -92,7 +91,7 @@ public class NodePrioritizer { */ void addSurplusNodes(List<Node> surplusNodes) { for (Node node : surplusNodes) { - PrioritizableNode nodePri = toPrioritizable(node, true, false); + NodeCandidate nodePri = candidateFrom(node, true, false); if (!nodePri.violatesSpares || isAllocatingForReplacement) { nodes.put(node, nodePri); } @@ -143,7 +142,7 @@ public class NodePrioritizer { resources(requestedNodes).with(host.flavor().resources().diskSpeed()) .with(host.flavor().resources().storageType()), NodeType.tenant); - PrioritizableNode nodePri = toPrioritizable(newNode, false, true); + NodeCandidate nodePri = candidateFrom(newNode, false, true); if ( ! nodePri.violatesSpares || isAllocatingForReplacement) { log.log(Level.FINE, "Adding new Docker node " + newNode); nodes.put(newNode, nodePri); @@ -159,7 +158,7 @@ public class NodePrioritizer { .filter(node -> legalStates.contains(node.state())) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(application)) - .map(node -> toPrioritizable(node, false, false)) + .map(node -> candidateFrom(node, false, false)) .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); } @@ -168,20 +167,17 @@ public class NodePrioritizer { allNodes.asList().stream() .filter(node -> node.type() == requestedNodes.type()) .filter(node -> node.state() == Node.State.ready) - .map(node -> toPrioritizable(node, false, false)) + .map(node -> candidateFrom(node, false, false)) .filter(n -> !n.violatesSpares || isAllocatingForReplacement) - .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); + .forEach(candidate -> nodes.put(candidate.node, candidate)); } - public List<PrioritizableNode> nodes() { return new ArrayList<>(nodes.values()); } + public List<NodeCandidate> nodes() { return new ArrayList<>(nodes.values()); } - /** - * Convert a list of nodes to a list of node priorities. This includes finding, calculating - * parameters to the priority sorting procedure. - */ - private PrioritizableNode toPrioritizable(Node node, boolean isSurplusNode, boolean isNewNode) { - PrioritizableNode.Builder builder = new PrioritizableNode.Builder(node).surplusNode(isSurplusNode) - .newNode(isNewNode); + /** Create a candidate from given node */ + private NodeCandidate candidateFrom(Node node, boolean isSurplusNode, boolean isNewNode) { + NodeCandidate.Builder builder = new NodeCandidate.Builder(node).surplusNode(isSurplusNode) + .newNode(isNewNode); allNodes.parentOf(node).ifPresent(parent -> { NodeResources parentCapacity = capacity.freeCapacityOf(parent, false); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 151fcb4233f..298404a4cb4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -38,7 +38,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { - var node = Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, Optional.empty(), NodeType.host); + var node = Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, Optional.empty(), NodeType.host, Optional.empty()); return node.with(node.status().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java new file mode 100644 index 00000000000..8692be2b322 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LocksResponse.java @@ -0,0 +1,106 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.curator.stats.LockAttempt; +import com.yahoo.vespa.curator.stats.LockCounters; +import com.yahoo.vespa.curator.stats.ThreadLockStats; + +import java.io.IOException; +import java.io.OutputStream; +import java.time.Instant; +import java.util.List; +import java.util.TreeMap; + +/** + * Returns information related to ZooKeeper locks. + * + * @author hakon + */ +public class LocksResponse extends HttpResponse { + + private final Slime slime = new Slime(); + + public LocksResponse() { + this(new TreeMap<>(ThreadLockStats.getLockCountersByPath()), + ThreadLockStats.getThreadLockStats(), + ThreadLockStats.getLockAttemptSamples()); + } + + /** For testing */ + LocksResponse(TreeMap<String, LockCounters> lockCountersByPath, + List<ThreadLockStats> threadLockStatsList, + List<LockAttempt> historicSamples) { + super(200); + + Cursor root = slime.setObject(); + + Cursor lockPathsCursor = root.setArray("lock-paths"); + lockCountersByPath.forEach((lockPath, lockCounters) -> { + Cursor lockPathCursor = lockPathsCursor.addObject(); + lockPathCursor.setString("path", lockPath); + lockPathCursor.setLong("in-critical-region", lockCounters.inCriticalRegionCount()); + lockPathCursor.setLong("invoke-acquire", lockCounters.invokeAcquireCount()); + lockPathCursor.setLong("acquire-failed", lockCounters.acquireFailedCount()); + lockPathCursor.setLong("acquire-timed-out", lockCounters.acquireTimedOutCount()); + lockPathCursor.setLong("lock-acquired", lockCounters.lockAcquiredCount()); + lockPathCursor.setLong("locks-released", lockCounters.locksReleasedCount()); + lockPathCursor.setLong("no-locks-errors", lockCounters.noLocksErrorCount()); + lockPathCursor.setLong("lock-release-errors", lockCounters.lockReleaseErrorCount()); + }); + + Cursor threadsCursor = root.setArray("threads"); + for (var threadLockStats : threadLockStatsList) { + List<LockAttempt> lockAttempts = threadLockStats.getLockAttempts(); + if (!lockAttempts.isEmpty()) { + Cursor threadLockStatsCursor = threadsCursor.addObject(); + threadLockStatsCursor.setString("thread-name", threadLockStats.getThreadName()); + + Cursor lockAttemptsCursor = threadLockStatsCursor.setArray("active-locks"); + for (var lockAttempt : lockAttempts) { + setLockAttempt(lockAttemptsCursor.addObject(), lockAttempt, false); + } + + threadLockStatsCursor.setString("stack-trace", threadLockStats.getStackTrace()); + } + } + + Cursor historicSamplesCursor = root.setArray("historic-samples"); + historicSamples.forEach(lockAttempt -> setLockAttempt(historicSamplesCursor.addObject(), lockAttempt, true)); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { + return "application/json"; + } + + private void setLockAttempt(Cursor lockAttemptCursor, LockAttempt lockAttempt, boolean includeThreadInfo) { + if (includeThreadInfo) { + lockAttemptCursor.setString("thread-name", lockAttempt.getThreadName()); + } + lockAttemptCursor.setString("lock-path", lockAttempt.getLockPath()); + lockAttemptCursor.setString("invoke-acquire-time", toString(lockAttempt.getTimeAcquiredWasInvoked())); + lockAttemptCursor.setString("acquire-timeout", lockAttempt.getAcquireTimeout().toString()); + lockAttempt.getTimeLockWasAcquired().ifPresent(instant -> lockAttemptCursor.setString("lock-acquired-time", toString(instant))); + lockAttemptCursor.setString("lock-state", lockAttempt.getLockState().name()); + lockAttempt.getTimeTerminalStateWasReached().ifPresent(instant -> lockAttemptCursor.setString("terminal-state-time", toString(instant))); + lockAttemptCursor.setString("acquire-duration", lockAttempt.getDurationOfAcquire().toString()); + lockAttemptCursor.setString("locked-duration", lockAttempt.getDurationWithLock().toString()); + lockAttemptCursor.setString("total-duration", lockAttempt.getDuration().toString()); + if (includeThreadInfo) { + lockAttempt.getStackTrace().ifPresent(stackTrace -> lockAttemptCursor.setString("stack-trace", stackTrace)); + } + } + + private static String toString(Instant time) { + return Instant.ofEpochMilli(time.toEpochMilli()).toString(); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 19276a81ef8..834bed00bd0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -149,6 +149,8 @@ public class NodePatcher { return patchRequiredDiskSpeed(node, asString(value)); case "reservedTo": return value.type() == Type.NIX ? node.withoutReservedTo() : node.withReservedTo(TenantName.from(value.asString())); + case "switchHostname": + return value.type() == Type.NIX ? node.withoutSwitchHostname() : node.withSwitchHostname(value.asString()); default : throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 8e3ae6358df..0137bee5fbd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -189,6 +189,7 @@ class NodesResponse extends HttpResponse { ipAddressesToSlime(node.ipConfig().pool().asSet(), object.setArray("additionalIpAddresses")); node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); + node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); } private void toSlime(ApplicationId id, Cursor object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 5080dafe2a5..a2d599eab6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -116,6 +116,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { if (pathS.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); if (pathS.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); if (pathS.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); + if (pathS.equals( "/nodes/v2/locks")) return new LocksResponse(); if (pathS.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); if (pathS.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); if (pathS.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); @@ -261,8 +262,13 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { modelName, flavorFromSlime(inspector), reservedToFromSlime(inspector.field("reservedTo")), - nodeTypeFromSlime(inspector.field("type")) - ); + nodeTypeFromSlime(inspector.field("type")), + switchHostnameFromSlime(inspector.field("switchHostname"))); + } + + private Optional<String> switchHostnameFromSlime(Inspector field) { + if (!field.valid()) return Optional.empty(); + return Optional.of(field.asString()); } private Flavor flavorFromSlime(Inspector inspector) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index b988a648632..9e51fa37246 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import static com.yahoo.config.provision.NodeResources.StorageType.local; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -51,7 +50,7 @@ class AutoscalingTester { public AutoscalingTester(NodeResources hostResources, HostResourcesCalculator resourcesCalculator) { this(new Zone(Environment.prod, RegionName.from("us-east")), List.of(new Flavor("hostFlavor", hostResources)), resourcesCalculator); provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); - provisioningTester.deployZoneApp(); + provisioningTester.activateTenantHosts(); } public AutoscalingTester(Zone zone, List<Flavor> flavors) { @@ -87,14 +86,14 @@ class AutoscalingTester { List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.from(new ClusterResources(nodes, groups, resources))); for (HostSpec host : hosts) makeReady(host.hostname()); - provisioningTester.deployZoneApp(); + provisioningTester.activateTenantHosts(); provisioningTester.activate(application, hosts); return hosts; } public void makeReady(String hostname) { Node node = nodeRepository().getNode(hostname).get(); - nodeRepository().write(node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())), nodeRepository().lock(node)); + provisioningTester.patchNode(node, (n) -> n.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of()))); Node host = nodeRepository().getNode(node.parentHostname().get()).get(); host = host.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); if (host.state() == Node.State.provisioned) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 379597827d9..c6809fd8369 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -16,14 +16,17 @@ import java.util.List; import static org.junit.Assert.assertEquals; +/** + * @author bratseth + */ public class NodeMetricsDbTest { @Test public void testNodeMetricsDb() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); - tester.makeReadyHosts(10, new NodeResources(10, 100, 1000, 10)).deployZoneApp(); + tester.makeReadyHosts(10, new NodeResources(10, 100, 1000, 10)) + .activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); - tester.deployZoneApp(); var hosts = tester.activate(app1, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("7.0").build(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java index e5273223b0e..ed6fa2c37b9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; -import com.yahoo.vdslib.state.NodeState; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; @@ -29,7 +28,7 @@ public class NodeMetricsFetcherTest { NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(), orchestrator, httpClient); tester.makeReadyNodes(4, resources); // Creates (in order) host-1.yahoo.com, host-2.yahoo.com, host-3.yahoo.com, host-4.yahoo.com - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application1 = ProvisioningTester.makeApplicationId(); ApplicationId application2 = ProvisioningTester.makeApplicationId(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 584cf24ac62..7cc81f218ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -179,7 +179,7 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); // Resume provisioning of new hosts List<Node> provisioned = tester.nodeRepository.list().state(Node.State.provisioned).asList(); tester.nodeRepository.setReady(provisioned, Agent.system, this.getClass().getSimpleName()); - tester.provisioningTester.deployZoneApp(); + tester.provisioningTester.activateTenantHosts(); // Allocating nodes to a host does not result in provisioning of additional capacity ApplicationId application = ProvisioningTester.makeApplicationId(); @@ -273,7 +273,8 @@ public class DynamicProvisioningMaintainerTest { false)); var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of()); return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(), - state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); + state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty(), + Optional.empty()); } private long provisionedHostsMatching(NodeResources resources) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 3d17cbf0217..056fe041377 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -133,7 +133,7 @@ public class InactiveAndFailedExpirerTest { // Flag one node for retirement and redeploy { Node toRetire = tester.getNodes(applicationId, Node.State.active).asList().get(0); - tester.patchNode(toRetire.withWantToRetire(true, Agent.operator, tester.clock().instant())); + tester.patchNode(toRetire, (node) -> node.withWantToRetire(true, Agent.operator, tester.clock().instant())); List<HostSpec> hostSpecs = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources))); tester.activate(applicationId, new HashSet<>(hostSpecs)); } @@ -203,9 +203,7 @@ public class InactiveAndFailedExpirerTest { assertEquals(2, inactiveNodes.size()); // Nodes marked for deprovisioning are moved to parked - tester.nodeRepository().write(inactiveNodes.stream() - .map(node -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())) - .collect(Collectors.toList()), () -> {}); + tester.patchNodes(inactiveNodes, (node) -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())); tester.advanceTime(Duration.ofMinutes(11)); new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(2, tester.nodeRepository().getNodes(Node.State.parked).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 20c9d24d1b6..872db9d62ca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -123,6 +123,15 @@ public class MetricsReporterTest { expectedMetrics.put("suspendedSeconds", 123L); expectedMetrics.put("numberOfServices", 0L); + expectedMetrics.put("cache.nodeObject.hitRate", 0.6D); + expectedMetrics.put("cache.nodeObject.evictionCount", 0L); + expectedMetrics.put("cache.nodeObject.size", 2L); + + nodeRepository.list(); + expectedMetrics.put("cache.curator.hitRate", 0.5D); + expectedMetrics.put("cache.curator.evictionCount", 0L); + expectedMetrics.put("cache.curator.size", 11L); + ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); @@ -163,7 +172,7 @@ public class MetricsReporterTest { Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost", - Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); nodeRepository.addNodes(List.of(dockerHost), Agent.system); nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName()); nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 435dcdf9223..21f836d2663 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -199,7 +199,8 @@ public class NodeFailTester { } public NodeFailer createFailer() { - return new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, downtimeLimitOneHour, clock, orchestrator, NodeFailer.ThrottlePolicy.hosted, metric); + return new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, downtimeLimitOneHour, + Duration.ofMinutes(5), clock, orchestrator, NodeFailer.ThrottlePolicy.hosted, metric); } public void allNodesMakeAConfigRequestExcept(Node ... deadNodeArray) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java index 218812f9a3d..f795dbaaa1c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java @@ -97,13 +97,7 @@ public class OsUpgradeActivatorTest { } private void completeUpgradeOf(List<Node> nodes) { - for (var node : nodes) { - try (var lock = tester.nodeRepository().lock(node)) { - node = tester.nodeRepository().getNode(node.hostname()).get(); - node = node.with(node.status().withVespaVersion(node.allocation().get().membership().cluster().vespaVersion())); - tester.nodeRepository().write(node, lock); - } - } + tester.patchNodes(nodes, (node) -> node.with(node.status().withVespaVersion(node.allocation().get().membership().cluster().vespaVersion()))); } private Stream<Node> streamUpdatedNodes(List<Node> nodes) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index ec830a7dc31..0ab6d4ddaa7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -316,14 +316,13 @@ public class PeriodicApplicationMaintainerTest { private List<Node> overriddenNodesNeedingMaintenance; - TestablePeriodicApplicationMaintainer setOverriddenNodesNeedingMaintenance(List<Node> overriddenNodesNeedingMaintenance) { + void setOverriddenNodesNeedingMaintenance(List<Node> overriddenNodesNeedingMaintenance) { this.overriddenNodesNeedingMaintenance = overriddenNodesNeedingMaintenance; - return this; } TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, Duration minTimeBetweenRedeployments) { - super(deployer, new TestMetric(), nodeRepository, interval, minTimeBetweenRedeployments); + super(deployer, new TestMetric(), nodeRepository, interval, minTimeBetweenRedeployments, new InMemoryFlagSource()); } @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 05e5b4829e9..65b79c2df04 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -54,7 +54,7 @@ public class RebalancerTest { // --- Making a more suitable node configuration available causes rebalancing Node newCpuHost = tester.makeReadyNode("cpu"); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.maintain(); assertTrue("Rebalancer retired the node we wanted to move away from", tester.isNodeRetired(cpuSkewedNode)); @@ -74,7 +74,7 @@ public class RebalancerTest { // --- Adding a more suitable node reconfiguration causes no action as the system is not stable Node memSkewedNode = tester.getNode(memoryApp); Node newMemHost = tester.makeReadyNode("mem"); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.maintain(); assertFalse("No rebalancing happens because cpuSkewedNode is still retired", tester.isNodeRetired(memSkewedNode)); @@ -117,7 +117,7 @@ public class RebalancerTest { // --- Making a more suitable node configuration available causes rebalancing Node newCpuHost = tester.makeReadyNode("cpu"); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.deployApp(cpuApp, false /* skip advancing clock after deployment */); tester.maintain(); @@ -152,7 +152,7 @@ public class RebalancerTest { deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); rebalancer = new Rebalancer(deployer, tester.nodeRepository(), metric, tester.clock(), Duration.ofMinutes(1)); tester.makeReadyNodes(3, "flat", NodeType.host, 8); - tester.deployZoneApp(); + tester.activateTenantHosts(); } void maintain() { rebalancer.maintain(); } @@ -163,7 +163,7 @@ public class RebalancerTest { NodeRepository nodeRepository() { return tester.nodeRepository(); } - void deployZoneApp() { tester.deployZoneApp(); } + void activateTenantHosts() { tester.activateTenantHosts(); } void deployApp(ApplicationId id) { deployApp(id, true); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index b7d6d7989e8..05b2d1e9ec9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -48,7 +48,7 @@ public class ScalingSuggestionsMaintainerTest { NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(tester.nodeRepository()); tester.makeReadyNodes(20, "flt", NodeType.host, 8); - tester.deployZoneApp(); + tester.activateTenantHosts(); tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index fcf2ab5a52d..bf83b074387 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -11,14 +10,15 @@ import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; -import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -59,10 +59,14 @@ public class IPTest { "::1", "::10", "::20", - "2001:db8:0:0:0:0:0:ffff", - "2001:db8:85a3:0:0:8a2e:370:7334", - "2001:db8:95a3:0:0:0:0:7334"), - new ArrayList<>(ImmutableSortedSet.copyOf(IP.NATURAL_ORDER, ipAddresses)) + "2001:db8::ffff", + "2001:db8:85a3::8a2e:370:7334", + "2001:db8:95a3::7334"), + ipAddresses.stream() + .map(IP::parse) + .sorted(IP.NATURAL_ORDER) + .map(IP::asString) + .collect(Collectors.toList()) ); } @@ -99,7 +103,6 @@ public class IPTest { @Test public void test_find_allocation_ipv4_only() { var pool = testPool(false); - assertTrue(pool instanceof IP.Ipv4Pool); var allocation = pool.findAllocation(emptyList, resolver); assertFalse("Found allocation", allocation.isEmpty()); assertEquals("127.0.0.1", allocation.get().primary()); @@ -173,13 +176,15 @@ public class IPTest { .addReverseRecord("::2", "host1"); } - return node.ipConfig().pool(); + IP.Pool pool = node.ipConfig().pool(); + assertNotEquals(dualStack, pool instanceof IP.Ipv4Pool); + return pool; } private static Node createNode(Set<String> ipAddresses) { return Node.create("id1", new IP.Config(Set.of("127.0.0.1"), ipAddresses), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - Optional.empty(), NodeType.host); + Optional.empty(), NodeType.host, Optional.empty()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 6a41e766ace..715ecdb5949 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; -import java.util.function.UnaryOperator; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -273,20 +272,11 @@ public class OsVersionsTest { } private void setWantedVersion(List<Node> nodes, Version wantedVersion) { - writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion))))); + tester.patchNodes(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion))))); } private void setCurrentVersion(List<Node> nodes, Version currentVersion) { - writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))))); - } - - private void writeNode(List<Node> nodes, UnaryOperator<Node> updateFunc) { - for (var node : nodes) { - try (var lock = tester.nodeRepository().lock(node)) { - node = tester.nodeRepository().getNode(node.hostname()).get(); - tester.nodeRepository().write(updateFunc.apply(node), lock); - } - } + tester.patchNodes(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))))); } private void completeUpgradeOf(List<Node> nodes) { @@ -294,7 +284,8 @@ public class OsVersionsTest { } private void completeUpgradeOf(List<Node> nodes, NodeType nodeType) { - writeNode(nodes, (node) -> { + // Complete upgrade by deprovisioning stale hosts and provisioning new ones + tester.patchNodes(nodes, (node) -> { Optional<Version> wantedOsVersion = node.status().osVersion().wanted(); if (node.status().wantToDeprovision()) { // Complete upgrade by deprovisioning stale hosts and provisioning new ones diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index b3d567c8d58..c7535f04c4f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -26,7 +26,7 @@ public class CuratorDatabaseClientTest { private final Curator curator = new MockCurator(); private final CuratorDatabaseClient zkClient = new CuratorDatabaseClient( - FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true, false); + FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true, false, 1000); @Test public void can_read_stored_host_information() throws Exception { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index dbbad0b8982..1c854068b27 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -53,11 +53,11 @@ import static org.junit.Assert.assertTrue; public class NodeSerializerTest { private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "large", "ugccloud-container"); - private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors); + private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors, 1000); private final ManualClock clock = new ManualClock(); @Test - public void testProvisionedNodeSerialization() { + public void provisioned_node_serialization() { Node node = createNode(); Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); @@ -69,7 +69,7 @@ public class NodeSerializerTest { } @Test - public void testReservedNodeSerialization() { + public void reserved_node_serialization() { Node node = createNode(); NodeResources requestedResources = new NodeResources(1.2, 3.4, 5.6, 7.8, NodeResources.DiskSpeed.any); @@ -110,7 +110,7 @@ public class NodeSerializerTest { } @Test - public void testRebootAndRestartAndTypeNoCurrentValuesSerialization() { + public void reboot_and_restart_and_type_no_current_values_serialization() { String nodeData = "{\n" + " \"type\" : \"tenant\",\n" + @@ -158,7 +158,7 @@ public class NodeSerializerTest { } @Test - public void testRetiredNodeSerialization() { + public void retired_node_serialization() { Node node = createNode(); clock.advance(Duration.ofMinutes(3)); @@ -185,7 +185,7 @@ public class NodeSerializerTest { } @Test - public void testAssimilatedDeserialization() { + public void assimilated_node_deserialization() { Node node = nodeSerializer.fromJson(Node.State.active, ("{\n" + " \"type\": \"tenant\",\n" + " \"hostname\": \"assimilate2.vespahosted.yahoo.tld\",\n" + @@ -211,7 +211,7 @@ public class NodeSerializerTest { } @Test - public void testSetFailCount() { + public void fail_count() { Node node = createNode(); node = node.allocate(ApplicationId.from(TenantName.from("myTenant"), ApplicationName.from("myApplication"), @@ -229,7 +229,7 @@ public class NodeSerializerTest { @Test public void serialize_parentHostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.tenant); + Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.tenant, Optional.empty()); Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); assertEquals(parentHostname, deserializedNode.parentHostname().get()); @@ -392,7 +392,7 @@ public class NodeSerializerTest { } @Test - public void testNodeWithNetworkPorts() { + public void network_ports_serialization() { Node node = createNode(); List<NetworkPorts.Allocation> list = new ArrayList<>(); list.add(new NetworkPorts.Allocation(8080, "container", "default/0", "http")); @@ -416,6 +416,15 @@ public class NodeSerializerTest { assertEquals(list, listCopy); } + @Test + public void switch_hostname_serialization() { + Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + assertFalse(node.switchHostname().isPresent()); + String switchHostname = "switch0.example.com"; + node = node.withSwitchHostname(switchHostname); + node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + assertEquals(switchHostname, node.switchHostname().get()); + } private byte[] createNodeJson(String hostname, String... ipAddress) { String ipAddressJsonPart = ""; @@ -441,8 +450,8 @@ public class NodeSerializerTest { Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - Optional.empty(), NodeType.tenant - ); + Optional.empty(), NodeType.tenant, + Optional.empty()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 86a2214f989..62c6c0c9426 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -165,7 +165,7 @@ public class AclProvisioningTest { public void trusted_nodes_for_application_with_load_balancer() { // Provision hosts and containers var hosts = tester.makeReadyNodes(2, "default", NodeType.host); - tester.deployZoneApp(); + tester.activateTenantHosts(); for (var host : hosts) { tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1), host.hostname()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index b113c281289..051420d694d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -80,7 +80,8 @@ public class AllocationSimulator { var ipConfig = new IP.Config(Set.of("127.0.0.1"), parent.isPresent() ? Set.of() : getAdditionalIP()); return new Node("fake", ipConfig, hostname, parent, flavor, Status.initial(), parent.isPresent() ? Node.State.ready : Node.State.active, allocation(tenant, flavor), History.empty(), - parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty(), Optional.empty()); + parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty(), Optional.empty(), + Optional.empty()); } private Set<String> getAdditionalIP() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java index cd6ae587b04..d5437296620 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java @@ -27,7 +27,7 @@ public class DockerImagesTest { // Host uses tenant default image (for preload purposes) var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); var hosts = tester.makeReadyNodes(2, "default", NodeType.host); - tester.deployZoneApp(); + tester.activateTenantHosts(); for (var host : hosts) { assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(host.type())); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java index d7ae651c891..d8ff34d1244 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningCompleteHostCalculatorTest.java @@ -30,7 +30,7 @@ public class DockerProvisioningCompleteHostCalculatorTest { .resourcesCalculator(new CompleteResourcesCalculator(hostFlavor)) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(9, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(9, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 5bd42908ab6..d2a5e06469a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -20,7 +20,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.NodeRepository; import org.junit.Test; import java.util.HashSet; @@ -123,7 +122,7 @@ public class DockerProvisioningTest { tester.makeReadyNodes(10, resources, Optional.of(tenant1), NodeType.host, 1); tester.makeReadyNodes(10, resources, Optional.empty(), NodeType.host, 1); - tester.deployZoneApp(); + tester.activateTenantHosts(); Version wantedVespaVersion = Version.fromString("6.39"); List<HostSpec> nodes = tester.prepare(application2_1, @@ -310,7 +309,7 @@ public class DockerProvisioningTest { .resourcesCalculator(3, 0) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(2, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(2, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -330,7 +329,7 @@ public class DockerProvisioningTest { .resourcesCalculator(3, 0) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(9, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(9, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -365,7 +364,7 @@ public class DockerProvisioningTest { .resourcesCalculator(3, 0) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(2, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(2, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -387,7 +386,7 @@ public class DockerProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(new Flavor(r))) .build(); - tester.makeReadyHosts(5, r).deployZoneApp(); + tester.makeReadyHosts(5, r).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -420,7 +419,7 @@ public class DockerProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(new Flavor(r))) .build(); - tester.makeReadyHosts(4, r).deployZoneApp(); + tester.makeReadyHosts(4, r).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(clusterType, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index b66dd3d4f7c..89072223341 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -66,7 +66,7 @@ public class DynamicDockerAllocationTest { .spareCount(spareCount) .build(); tester.makeReadyNodes(4, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources flavor = new NodeResources(1, 4, 100, 1); @@ -109,7 +109,7 @@ public class DynamicDockerAllocationTest { public void relocate_failed_nodes() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources resources = new NodeResources(1, 4, 100, 0.3); @@ -158,7 +158,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(3, "flt", NodeType.host, 8); // cpu: 30, mem: 30 tester.makeReadyNodes(3, "cpu", NodeType.host, 8); // cpu: 40, mem: 20 tester.makeReadyNodes(3, "mem", NodeType.host, 8); // cpu: 20, mem: 40 - tester.deployZoneApp(); + tester.activateTenantHosts(); NodeResources fltResources = new NodeResources(6, 6, 10, 0.1); NodeResources cpuResources = new NodeResources(8, 4, 10, 0.1); NodeResources memResources = new NodeResources(4, 8, 10, 0.1); @@ -201,7 +201,7 @@ public class DynamicDockerAllocationTest { public void do_not_relocate_nodes_from_spare_if_no_where_to_relocate_them() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources flavor = new NodeResources(1, 4, 100, 1); @@ -228,7 +228,7 @@ public class DynamicDockerAllocationTest { public void multiple_groups_are_on_separate_parent_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); //Deploy an application having 6 nodes (3 nodes in 2 groups). We only have 5 docker hosts available ApplicationId application1 = ProvisioningTester.makeApplicationId(); @@ -249,7 +249,7 @@ public class DynamicDockerAllocationTest { // Setup test ApplicationId application1 = ProvisioningTester.makeApplicationId(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); NodeResources flavor = new NodeResources(1, 4, 100, 1); // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) @@ -278,7 +278,7 @@ public class DynamicDockerAllocationTest { public void non_prod_zones_do_not_have_spares() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(3, "host-small", NodeType.host, 32); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application1 = ProvisioningTester.makeApplicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); @@ -291,7 +291,7 @@ public class DynamicDockerAllocationTest { public void cd_uses_slow_disk_nodes_for_docker_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(SystemName.cd, Environment.test, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(4, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application1 = ProvisioningTester.makeApplicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); @@ -311,7 +311,7 @@ public class DynamicDockerAllocationTest { public void provision_dual_stack_containers() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, "host-large", NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); List<HostSpec> hosts = tester.prepare(application, clusterSpec("myContent.t1.a1"), 2, 1, new NodeResources(1, 4, 100, 1)); @@ -342,7 +342,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); @@ -359,7 +359,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); @@ -381,7 +381,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); @@ -399,7 +399,7 @@ public class DynamicDockerAllocationTest { public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); @@ -425,7 +425,7 @@ public class DynamicDockerAllocationTest { private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, NodeResources flavor, int index, ProvisioningTester tester) { Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, - Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), Optional.empty(), NodeType.tenant + Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), Optional.empty(), NodeType.tenant, Optional.empty() ); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index a70347f6316..84850358798 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -81,7 +81,7 @@ public class DynamicDockerProvisionTest { @Test public void does_not_allocate_to_available_empty_hosts() { tester.makeReadyNodes(3, "small", NodeType.host, 10); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); NodeResources flavor = new NodeResources(1, 4, 10, 1); @@ -109,7 +109,7 @@ public class DynamicDockerProvisionTest { tester.nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); nameResolver.addRecord(hostname + "-2", "::" + i + ":2"); } - tester.deployZoneApp(); + tester.activateTenantHosts(); mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("small")); tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor); @@ -128,15 +128,14 @@ public class DynamicDockerProvisionTest { NodeResources resources = new NodeResources(10, 10, 10, 10); ApplicationId app = ProvisioningTester.makeApplicationId(); - Function<Node, Node> retireNode = node -> - tester.nodeRepository().write(node.withWantToRetire(true, Agent.system, Instant.now()), () -> {}); + Function<Node, Node> retireNode = node -> tester.patchNode(node, (n) -> n.withWantToRetire(true, Agent.system, Instant.now())); Function<Integer, Node> getNodeInGroup = group -> tester.nodeRepository().getNodes(app).stream() .filter(node -> node.allocation().get().membership().cluster().group().get().index() == group) .findAny().orElseThrow(); // Allocate 10 hosts tester.makeReadyNodes(10, resources, NodeType.host, 1); - tester.deployZoneApp(); + tester.activateTenantHosts(); // Prepare & activate an application with 8 nodes and 2 groups tester.activate(app, tester.prepare(app, clusterSpec("content"), 8, 2, resources)); @@ -170,7 +169,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, 0) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -216,7 +215,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, 0) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -291,7 +290,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, 0) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -326,7 +325,7 @@ public class DynamicDockerProvisionTest { .resourcesCalculator(memoryTax, localDiskTax) .build(); - tester.deployZoneApp(); + tester.activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index da78aff493e..f831f2d501b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -44,9 +44,9 @@ public class HostCapacityTest { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); // Create three docker hosts - host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); - host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); - host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); + host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); + host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); // Add two containers to host1 var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant); @@ -111,7 +111,7 @@ public class HostCapacityTest { // Dev host can assign both configserver and tenant containers. var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); - var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), Optional.empty(), NodeType.devhost); + var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), Optional.empty(), NodeType.devhost, Optional.empty()); var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index ae8753afa2d..aba7ac2a530 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -134,7 +134,7 @@ public class MultigroupProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(6, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(6, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -160,7 +160,7 @@ public class MultigroupProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(6, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(6, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -190,7 +190,7 @@ public class MultigroupProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(12, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(12, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java index 3865baa51c1..ec001556f58 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -22,29 +22,29 @@ import static org.junit.Assert.assertEquals; /** * @author bratseth */ -public class PrioritizableNodeTest { +public class NodeCandidateTest { @Test public void test_order() { - List<PrioritizableNode> expected = List.of( - new PrioritizableNode(node("01", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, false, false), - new PrioritizableNode(node("02", Node.State.active), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), - new PrioritizableNode(node("04", Node.State.reserved), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), - new PrioritizableNode(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), - new PrioritizableNode(node("05", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, false, true, false), - new PrioritizableNode(node("06", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, false, true, false), - new PrioritizableNode(node("07", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, false, true, false), - new PrioritizableNode(node("08", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, false, true, false), - new PrioritizableNode(node("09", Node.State.ready), new NodeResources(1, 1, 1, 1), Optional.empty(), true, false, true, false), - new PrioritizableNode(node("10", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false), - new PrioritizableNode(node("11", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false) + List<NodeCandidate> expected = List.of( + new NodeCandidate(node("01", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, false, false), + new NodeCandidate(node("02", Node.State.active), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), + new NodeCandidate(node("04", Node.State.reserved), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), + new NodeCandidate(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), + new NodeCandidate(node("05", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, false, true, false), + new NodeCandidate(node("06", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, false, true, false), + new NodeCandidate(node("07", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, false, true, false), + new NodeCandidate(node("08", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, false, true, false), + new NodeCandidate(node("09", Node.State.ready), new NodeResources(1, 1, 1, 1), Optional.empty(), true, false, true, false), + new NodeCandidate(node("10", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false), + new NodeCandidate(node("11", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, true, false) ); assertOrder(expected); } @Test public void testOrderingByAllocationSkew1() { - List<PrioritizableNode> expected = List.of( + List<NodeCandidate> expected = List.of( node("1", node(4, 4), host(20, 20), host(40, 40)), node("2", node(4, 4), host(21, 20), host(40, 40)), node("3", node(4, 4), host(22, 20), host(40, 40)), @@ -59,7 +59,7 @@ public class PrioritizableNodeTest { // The same as testOrderingByAllocationSkew1, but deviating from mean (20) in the other direction. // Since we don't choose the node with the lowest skew, but with the largest skew *reduction* // this causes the opposite order. - List<PrioritizableNode> expected = List.of( + List<NodeCandidate> expected = List.of( node("4", node(4, 4), host(19, 18), host(40, 40)), node("3", node(4, 4), host(18, 20), host(40, 40)), node("2", node(4, 4), host(19, 20), host(40, 40)), @@ -72,7 +72,7 @@ public class PrioritizableNodeTest { @Test public void testOrderingByAllocationSkew3() { // The same as testOrderingByAllocationSkew1, but allocating skewed towards cpu - List<PrioritizableNode> expected = List.of( + List<NodeCandidate> expected = List.of( node("1", node(4, 2), host(20, 20), host(40, 40)), node("2", node(4, 2), host(21, 20), host(40, 40)), node("4", node(4, 2), host(21, 22), host(40, 40)), @@ -85,7 +85,7 @@ public class PrioritizableNodeTest { @Test public void testOrderingByAllocationSkew4() { // The same as testOrderingByAllocationSkew1, but allocating skewed towards memory - List<PrioritizableNode> expected = List.of( + List<NodeCandidate> expected = List.of( node("5", node(2, 10), host(21, 21), host(40, 80)), node("3", node(2, 10), host(22, 20), host(40, 40)), node("2", node(2, 10), host(21, 20), host(40, 40)), @@ -99,7 +99,7 @@ public class PrioritizableNodeTest { public void testOrderingByAllocationSkew5() { // node1 is skewed towards cpu (without this allocation), allocation is skewed towards memory, therefore // node 1 is preferred (even though it is still most skewed) - List<PrioritizableNode> expected = List.of( + List<NodeCandidate> expected = List.of( node("1", node(1, 5), host(21, 10), host(40, 40)), node("2", node(1, 5), host(21, 20), host(40, 40)), node("3", node(1, 5), host(20, 20), host(40, 40)), @@ -108,8 +108,8 @@ public class PrioritizableNodeTest { assertOrder(expected); } - private void assertOrder(List<PrioritizableNode> expected) { - List<PrioritizableNode> copy = new ArrayList<>(expected); + private void assertOrder(List<NodeCandidate> expected) { + List<NodeCandidate> copy = new ArrayList<>(expected); Collections.shuffle(copy); Collections.sort(copy); assertEquals(expected, copy); @@ -127,23 +127,23 @@ public class PrioritizableNodeTest { return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(new NodeResources(2, 2, 2, 2)), Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), - Optional.empty(), Optional.empty()); + Optional.empty(), Optional.empty(), Optional.empty()); } - private static PrioritizableNode node(String hostname, - NodeResources nodeResources, - NodeResources allocatedHostResources, // allocated before adding nodeResources - NodeResources totalHostResources) { + private static NodeCandidate node(String hostname, + NodeResources nodeResources, + NodeResources allocatedHostResources, // allocated before adding nodeResources + NodeResources totalHostResources) { Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), new Flavor(nodeResources), Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, - new Reports(), Optional.empty(), Optional.empty()); + new Reports(), Optional.empty(), Optional.empty(), Optional.empty()); Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(totalHostResources), Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, - new Reports(), Optional.empty(), Optional.empty()); - return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), - false, false, true, false); + new Reports(), Optional.empty(), Optional.empty(), Optional.empty()); + return new NodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), + false, false, true, false); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index d5d7a73992e..1c03bbc6a56 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -62,7 +62,7 @@ public class ProvisioningTest { ApplicationId application1 = ProvisioningTester.makeApplicationId(); ApplicationId application2 = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(21, defaultResources).deployZoneApp(); + tester.makeReadyHosts(21, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -196,7 +196,7 @@ public class ProvisioningTest { ApplicationId application1 = ProvisioningTester.makeApplicationId(); tester.makeReadyHosts(24, defaultResources); - tester.deployZoneApp(); + tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -267,7 +267,7 @@ public class ProvisioningTest { ApplicationId application1 = ProvisioningTester.makeApplicationId(); tester.makeReadyHosts(12, small); - tester.deployZoneApp(); + tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); @@ -278,7 +278,7 @@ public class ProvisioningTest { tester.activate(application1, state2.allHosts); tester.makeReadyHosts(16, large); - tester.deployZoneApp(); + tester.activateTenantHosts(); // redeploy with increased sizes and new flavor SystemState state3 = prepare(application1, 3, 4, 4, 5, large, tester); @@ -303,7 +303,7 @@ public class ProvisioningTest { tester.makeReadyHosts(12, small); tester.makeReadyHosts(12, large); - tester.deployZoneApp(); + tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); @@ -319,7 +319,7 @@ public class ProvisioningTest { ApplicationId application1 = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(5, defaultResources).deployZoneApp(); + tester.makeReadyHosts(5, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 0, 3, 0, defaultResources, tester); @@ -354,7 +354,7 @@ public class ProvisioningTest { @Test public void requested_resources_info_is_retained() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(13, defaultResources).deployZoneApp(); + tester.makeReadyHosts(13, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(ProvisioningTester.makeApplicationId(), NodeType.host); ApplicationId application = ProvisioningTester.makeApplicationId(); @@ -398,7 +398,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(ProvisioningTester.makeApplicationId(), NodeType.host); ApplicationId application = ProvisioningTester.makeApplicationId(); @@ -411,7 +411,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version_and_docker_image() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(ProvisioningTester.makeApplicationId(), NodeType.host); ApplicationId application = ProvisioningTester.makeApplicationId(); @@ -426,7 +426,8 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); + SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -438,7 +439,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(4, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(4, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -456,7 +457,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(31, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(31, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -516,7 +517,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); prepare(application, 1, 2, 3, 3, defaultResources, tester); } @@ -525,7 +526,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); try { prepare(application, 2, 2, 3, 3, new NodeResources(2, 2, 10, 2), tester); @@ -540,7 +541,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); try { prepare(application, 2, 2, 3, 3, new NodeResources(0.4, 4, 10, 2), tester); @@ -573,7 +574,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(4, large).deployZoneApp(); + tester.makeReadyHosts(4, large).activateTenantHosts(); SystemState state = prepare(application, 2, 2, 3, 3, large, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -584,7 +585,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(14, defaultResources).deployZoneApp(); + tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6 assertEquals(9, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -594,7 +595,7 @@ public class ProvisioningTest { public void activate_after_reservation_timeout() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -616,7 +617,7 @@ public class ProvisioningTest { public void out_of_capacity() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(9, defaultResources).deployZoneApp(); // need 2+2+3+3=10 + tester.makeReadyHosts(9, defaultResources).activateTenantHosts(); // need 2+2+3+3=10 ApplicationId application = ProvisioningTester.makeApplicationId(); try { prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -633,7 +634,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(13, defaultResources).deployZoneApp(); + tester.makeReadyHosts(13, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); try { prepare(application, 2, 2, 6, 3, defaultResources, tester); @@ -651,7 +652,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(13, defaultResources).deployZoneApp(); + tester.makeReadyHosts(13, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); prepare(application, 2, 2, 6, 3, defaultResources, tester); } @@ -659,7 +660,7 @@ public class ProvisioningTest { @Test public void out_of_capacity_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified()), false, false)); @@ -673,7 +674,7 @@ public class ProvisioningTest { ApplicationId application = ProvisioningTester.makeApplicationId(); // Flag all nodes for retirement List<Node> readyNodes = tester.makeReadyNodes(5, defaultResources); - readyNodes.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); + tester.patchNodes(readyNodes, (node) -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); try { prepare(application, 2, 0, 2, 0, defaultResources, tester); @@ -693,7 +694,7 @@ public class ProvisioningTest { ApplicationId application = ProvisioningTester.makeApplicationId(); // Create 10 nodes - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); // Allocate 5 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.activate(application, tester.prepare(application, cluster, capacity)); @@ -701,7 +702,7 @@ public class ProvisioningTest { assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); // Mark the nodes as want to retire - tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); + tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node, (n) -> n.withWantToRetire(true, Agent.system, tester.clock().instant()))); // redeploy without allow failing tester.activate(application, tester.prepare(application, cluster, capacityFORCED)); @@ -724,7 +725,7 @@ public class ProvisioningTest { ApplicationId application1 = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(14, defaultResources).deployZoneApp(); + tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 3, 3, 4, 4, defaultResources, tester); @@ -751,7 +752,7 @@ public class ProvisioningTest { public void node_on_spare_host_retired_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .spareCount(1).build(); - tester.makeReadyHosts(7, defaultResources).deployZoneApp(); + tester.makeReadyHosts(7, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.makeApplicationId(); ClusterSpec spec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7.1.2").build(); @@ -774,7 +775,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(10, defaultResources).deployZoneApp(); + tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); // Deploy application { @@ -786,7 +787,7 @@ public class ProvisioningTest { // Retire some nodes and redeploy { List<Node> nodesToRetire = tester.getNodes(application, Node.State.active).asList().subList(0, 2); - nodesToRetire.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); + tester.patchNodes(nodesToRetire, (node) -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester); tester.activate(application, state.allHosts); @@ -802,7 +803,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(2, defaultResources).deployZoneApp(); + tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); // Deploy fails with out of capacity try { @@ -813,7 +814,7 @@ public class ProvisioningTest { tester.getNodes(application, Node.State.reserved).size()); // Enough nodes become available - tester.makeReadyHosts(2, defaultResources).deployZoneApp(); + tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); // Deploy is retried after a few minutes tester.clock().advance(Duration.ofMinutes(2)); @@ -883,7 +884,7 @@ public class ProvisioningTest { var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); var application = ProvisioningTester.makeApplicationId(); - tester.makeReadyHosts(4, defaultResources).deployZoneApp(); + tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index e7e2ca291a0..078b1d372df 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -53,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Function; +import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.stream.Collectors; @@ -139,7 +140,22 @@ public class ProvisioningTester { public CapacityPolicies capacityPolicies() { return capacityPolicies; } public NodeList getNodes(ApplicationId id, Node.State ... inState) { return NodeList.copyOf(nodeRepository.getNodes(id, inState)); } - public void patchNode(Node node) { nodeRepository.write(node, () -> {}); } + public Node patchNode(Node node, UnaryOperator<Node> patcher) { + return patchNodes(List.of(node), patcher).get(0); + } + + public List<Node> patchNodes(List<Node> nodes, UnaryOperator<Node> patcher) { + List<Node> updated = new ArrayList<>(); + for (var node : nodes) { + try (var lock = nodeRepository.lock(node)) { + node = nodeRepository.getNode(node.hostname()).get(); + node = patcher.apply(node); + nodeRepository.write(node, lock); + updated.add(node); + } + } + return updated; + } public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, NodeResources resources) { return prepare(application, cluster, nodeCount, groups, false, resources); @@ -320,7 +336,7 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } - /** Call deployZoneApp() after this before deploying applications */ + /** Call {@link this#activateTenantHosts()} after this before deploying applications */ public ProvisioningTester makeReadyHosts(int n, NodeResources resources) { makeReadyNodes(n, resources, NodeType.host, 5); return this; @@ -494,7 +510,7 @@ public class ProvisioningTester { return nodes; } - public void deployZoneApp() { + public void activateTenantHosts() { ApplicationId applicationId = makeApplicationId(); List<HostSpec> list = prepare(applicationId, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 874db8b47df..b5e31e7cbdb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -129,9 +129,7 @@ public class VirtualNodeProvisioningTest { assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount); assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); - for (Node n : nodes) { - tester.patchNode(n.withParentHostname("clashing")); - } + tester.patchNodes(nodes, (n) -> n.withParentHostname("clashing")); containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); contentHosts = prepare(contentClusterSpec, contentNodeCount, groups); activate(containerHosts, contentHosts); @@ -160,9 +158,7 @@ public class VirtualNodeProvisioningTest { assertDistinctParentHosts(nodes, ClusterSpec.Type.container, containerNodeCount); assertDistinctParentHosts(nodes, ClusterSpec.Type.content, contentNodeCount); - for (Node n : nodes) { - tester.patchNode(n.withParentHostname("clashing")); - } + tester.patchNodes(nodes, (n) -> n.withParentHostname("clashing")); OutOfCapacityException expected = null; try { containerHosts = prepare(containerClusterSpec, containerNodeCount, groups); @@ -216,9 +212,9 @@ public class VirtualNodeProvisioningTest { assertEquals(3, nodes.size()); // Set indistinct parents - tester.patchNode(nodes.get(0).withParentHostname("parentHost1")); - tester.patchNode(nodes.get(1).withParentHostname("parentHost1")); - tester.patchNode(nodes.get(2).withParentHostname("parentHost2")); + tester.patchNode(nodes.get(0), (n) -> n.withParentHostname("parentHost1")); + tester.patchNode(nodes.get(1), (n) -> n.withParentHostname("parentHost1")); + tester.patchNode(nodes.get(2), (n) -> n.withParentHostname("parentHost2")); nodes = getNodes(applicationId); assertEquals(3, nodes.stream().filter(n -> n.parentHostname().isPresent()).count()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 2cd2fe1fc28..9f6f5043ae8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -305,7 +305,7 @@ public class NodesV2ApiTest { ("[" + asNodeJson("host-with-ip.yahoo.com", "default", "foo") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST); - tester.assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); + tester.assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid IP address 'foo': 'foo' is not an IP string literal.\"}"); // Attempt to POST tenant node with already assigned IP tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -333,7 +333,7 @@ public class NodesV2ApiTest { // Node types running a single container can share their IP address with child node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), "127.0.42.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -349,7 +349,7 @@ public class NodesV2ApiTest { // ... nor with child node on different host tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), "127.0.43.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", @@ -939,6 +939,30 @@ public class NodesV2ApiTest { "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); } + @Test + public void test_node_switch_hostname() throws Exception { + String hostname = "host42.yahoo.com"; + // Add host with switch hostname + String json = asNodeJson(hostname, NodeType.host, "default", Optional.empty(), Optional.of("switch0"), "127.0.42.1", "::42:1"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + json + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), "\"switchHostname\":\"switch0\""); + + // Update switch hostname + json = "{\"switchHostname\":\"switch1\"}"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, json.getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), + "{\"message\":\"Updated host42.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), "\"switchHostname\":\"switch1\""); + + // Clear switch hostname + json = "{\"switchHostname\":null}"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, json.getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), + "{\"message\":\"Updated host42.yahoo.com\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname), "switchHostname", false); + } + private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { return asDockerNodeJson(hostname, NodeType.tenant, parentHostname, ipAddress); } @@ -958,14 +982,15 @@ public class NodesV2ApiTest { } private static String asHostJson(String hostname, String flavor, Optional<TenantName> reservedTo, String... ipAddress) { - return asNodeJson(hostname, NodeType.host, flavor, reservedTo, ipAddress); + return asNodeJson(hostname, NodeType.host, flavor, reservedTo, Optional.empty(), ipAddress); } - private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, String... ipAddress) { + private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, Optional<String> switchHostname, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + createIpAddresses(ipAddress) + "\"flavor\":\"" + flavor + "\"" + (reservedTo.isPresent() ? ", \"reservedTo\":\"" + reservedTo.get().value() + "\"" : "") + + (switchHostname.isPresent() ? ", \"switchHostname\":\"" + switchHostname.get() + "\"" : "") + ", \"type\":\"" + nodeType + "\"}"; } @@ -989,8 +1014,8 @@ public class NodesV2ApiTest { tester.assertFile(request, file); } - private void assertResponse(Request request, String file) throws IOException { - tester.assertResponse(request, file); + private void assertResponse(Request request, String response) throws IOException { + tester.assertResponse(request, response); } } diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index f29712e7a5f..f98a3c87a2e 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -46,9 +46,9 @@ vespa_define_module( src/apps/tests src/apps/verify_ranksetup src/apps/vespa-dump-feed + src/apps/vespa-feed-bm src/apps/vespa-gen-testdocs src/apps/vespa-proton-cmd - src/apps/vespa-spi-feed-bm src/apps/vespa-transactionlog-inspect TESTS diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 2d028f47513..1d492cb558f 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -59,6 +59,7 @@ OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupC for (const auto &entry: modelsCfg.model) { if (auto file = get_file(entry.fileref, myCfg)) { model_list.emplace_back(entry.name, file.value()); + OnnxModels::configure(entry, model_list.back()); } else { LOG(warning, "could not find file for onnx model '%s' (ref:'%s')\n", entry.name.c_str(), entry.fileref.c_str()); diff --git a/searchcore/src/apps/vespa-feed-bm/.gitignore b/searchcore/src/apps/vespa-feed-bm/.gitignore new file mode 100644 index 00000000000..0dc27e95ea8 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/.gitignore @@ -0,0 +1 @@ +vespa-feed-bm diff --git a/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt index e188bc16ec0..4ced3fe173b 100644 --- a/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt +++ b/searchcore/src/apps/vespa-feed-bm/CMakeLists.txt @@ -1,8 +1,11 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_vespa_spi_feed_bm_app +vespa_add_executable(searchcore_vespa_feed_bm_app SOURCES - vespa_spi_feed_bm.cpp - OUTPUT_NAME vespa-spi-feed-bm + vespa_feed_bm.cpp + spi_bm_feed_handler.cpp + storage_api_rpc_bm_feed_handler.cpp + storage_api_chain_bm_feed_handler.cpp + OUTPUT_NAME vespa-feed-bm DEPENDS searchcore_server searchcore_initializer @@ -20,5 +23,8 @@ vespa_add_executable(searchcore_vespa_spi_feed_bm_app searchcore_grouping searchcore_proton_metrics searchcore_fconfig + storageserver_storageapp + messagebus_messagebus-test + messagebus searchlib_searchlib_uca ) diff --git a/searchcore/src/apps/vespa-feed-bm/i_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/i_bm_feed_handler.h new file mode 100644 index 00000000000..a3341bf14c9 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/i_bm_feed_handler.h @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace document { +class Bucket; +class Document; +class DocumentUpdate; +class DocumentId; +} + +namespace feedbm { + +class PendingTracker; + +/* + * Interface class for benchmark feed handler. + */ +class IBmFeedHandler +{ +public: + virtual ~IBmFeedHandler() = default; + virtual void put(const document::Bucket& bucket, std::unique_ptr<document::Document> document, uint64_t timestamp, PendingTracker& tracker) = 0; + virtual void update(const document::Bucket& bucket, std::unique_ptr<document::DocumentUpdate> document_update, uint64_t timestamp, PendingTracker& tracker) = 0; + virtual void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) = 0; +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/pending_tracker.h b/searchcore/src/apps/vespa-feed-bm/pending_tracker.h new file mode 100644 index 00000000000..3698832068f --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/pending_tracker.h @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <mutex> +#include <condition_variable> + +namespace feedbm { + +/* + * Class to track number of pending operations, used as backpressure during + * benchmark feeding. + */ +class PendingTracker { + uint32_t _pending; + uint32_t _limit; + std::mutex _mutex; + std::condition_variable _cond; + +public: + PendingTracker(uint32_t limit) + : _pending(0u), + _limit(limit), + _mutex(), + _cond() + { + } + + ~PendingTracker() + { + drain(); + } + + void release() { + std::unique_lock<std::mutex> guard(_mutex); + --_pending; + if (_pending < _limit) { + _cond.notify_all(); + } + } + void retain() { + std::unique_lock<std::mutex> guard(_mutex); + while (_pending >= _limit) { + _cond.wait(guard); + } + ++_pending; + } + + void drain() { + std::unique_lock<std::mutex> guard(_mutex); + while (_pending > 0) { + _cond.wait(guard); + } + } +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp new file mode 100644 index 00000000000..d53ece2fc42 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.cpp @@ -0,0 +1,93 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "spi_bm_feed_handler.h" +#include "pending_tracker.h" +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/metrics/loadtype.h> +#include <vespa/persistence/spi/persistenceprovider.h> + +using document::Document; +using document::DocumentId; +using document::DocumentUpdate; +using storage::spi::Bucket; +using storage::spi::PartitionId; +using storage::spi::PersistenceProvider; +using storage::spi::Timestamp; + +namespace feedbm { + +namespace { + +storage::spi::LoadType default_load_type(0, "default"); +storage::spi::Context context(default_load_type, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + +class MyOperationComplete : public storage::spi::OperationComplete +{ + PendingTracker& _tracker; +public: + MyOperationComplete(PendingTracker& tracker); + ~MyOperationComplete(); + void onComplete(std::unique_ptr<storage::spi::Result> result) override; + void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; +}; + +MyOperationComplete::MyOperationComplete(PendingTracker& tracker) + : _tracker(tracker) +{ + _tracker.retain(); +} + +MyOperationComplete::~MyOperationComplete() +{ + _tracker.release(); +} + +void +MyOperationComplete::onComplete(std::unique_ptr<storage::spi::Result> result) +{ + (void) result; +} + +void +MyOperationComplete::addResultHandler(const storage::spi::ResultHandler * resultHandler) +{ + (void) resultHandler; +} + +} + +SpiBmFeedHandler::SpiBmFeedHandler(PersistenceProvider& provider) + : IBmFeedHandler(), + _provider(provider) +{ +} + +SpiBmFeedHandler::~SpiBmFeedHandler() = default; + +void +SpiBmFeedHandler::put(const document::Bucket& bucket, std::unique_ptr<Document> document, uint64_t timestamp, PendingTracker& tracker) +{ + _provider.putAsync(Bucket(bucket, PartitionId(0)), Timestamp(timestamp), std::move(document), context, std::make_unique<MyOperationComplete>(tracker)); +} + +void +SpiBmFeedHandler::update(const document::Bucket& bucket, std::unique_ptr<DocumentUpdate> document_update, uint64_t timestamp, PendingTracker& tracker) +{ + _provider.updateAsync(Bucket(bucket, PartitionId(0)), Timestamp(timestamp), std::move(document_update), context, std::make_unique<MyOperationComplete>(tracker)); +} + +void +SpiBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) +{ + _provider.removeAsync(Bucket(bucket, PartitionId(0)), Timestamp(timestamp), document_id, context, std::make_unique<MyOperationComplete>(tracker)); + +} + +void +SpiBmFeedHandler::create_bucket(const document::Bucket& bucket) +{ + _provider.createBucket(Bucket(bucket, PartitionId(0)), context); +} + +} diff --git a/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.h new file mode 100644 index 00000000000..5b56a4f21dd --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/spi_bm_feed_handler.h @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_bm_feed_handler.h" + +namespace storage::spi { struct PersistenceProvider; } + +namespace feedbm { + +/* + * Benchmark feed handler for feed directly to persistence provider + */ +class SpiBmFeedHandler : public IBmFeedHandler +{ + storage::spi::PersistenceProvider& _provider; +public: + SpiBmFeedHandler(storage::spi::PersistenceProvider& provider); + ~SpiBmFeedHandler(); + void put(const document::Bucket& bucket, std::unique_ptr<document::Document> document, uint64_t timestamp, PendingTracker& tracker) override; + void update(const document::Bucket& bucket, std::unique_ptr<document::DocumentUpdate> document_update, uint64_t timestamp, PendingTracker& tracker) override; + void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) override; + void create_bucket(const document::Bucket& bucket); +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp new file mode 100644 index 00000000000..6f1acd10fe4 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.cpp @@ -0,0 +1,185 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_api_chain_bm_feed_handler.h" +#include "pending_tracker.h" +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storage/common/storagelink.h> +#include <vespa/storage/common/storage_chain_builder.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> +#include <cassert> + +using document::Document; +using document::DocumentId; +using document::DocumentUpdate; +using storage::StorageLink; + +namespace feedbm { + +namespace { + +std::shared_ptr<storage::api::StorageCommand> make_set_cluster_state_cmd() { + storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); + auto cmd = std::make_shared<storage::api::SetSystemStateCommand>(bundle); + cmd->setPriority(storage::api::StorageMessage::VERYHIGH); + return cmd; +} + +} + +class BmStorageLink : public StorageLink +{ + std::mutex _mutex; + vespalib::hash_map<uint64_t, PendingTracker *> _pending; +public: + BmStorageLink(); + ~BmStorageLink() override; + bool onDown(const std::shared_ptr<storage::api::StorageMessage>& msg) override; + bool onUp(const std::shared_ptr<storage::api::StorageMessage>& msg) override; + void retain(uint64_t msg_id, PendingTracker &tracker) { + tracker.retain(); + std::lock_guard lock(_mutex); + _pending.insert(std::make_pair(msg_id, &tracker)); + } + bool release(uint64_t msg_id) { + PendingTracker *tracker = nullptr; + { + std::lock_guard lock(_mutex); + auto itr = _pending.find(msg_id); + if (itr == _pending.end()) { + return false; + } + tracker = itr->second; + _pending.erase(itr); + } + tracker->release(); + return true; + } +}; + +BmStorageLink::BmStorageLink() + : storage::StorageLink("vespa-bm-feed"), + _mutex(), + _pending() +{ +} + +BmStorageLink::~BmStorageLink() +{ + std::lock_guard lock(_mutex); + assert(_pending.empty()); +} + +bool +BmStorageLink::onDown(const std::shared_ptr<storage::api::StorageMessage>& msg) +{ + (void) msg; + return false; +} + +bool +BmStorageLink::onUp(const std::shared_ptr<storage::api::StorageMessage>& msg) +{ + return release(msg->getMsgId()); +} + +struct StorageApiChainBmFeedHandler::Context { + BmStorageLink* bm_link; + Context() + : bm_link(nullptr) + { + } + ~Context() = default; +}; + +class MyStorageChainBuilder : public storage::StorageChainBuilder +{ + using Parent = storage::StorageChainBuilder; + std::shared_ptr<StorageApiChainBmFeedHandler::Context> _context; +public: + MyStorageChainBuilder(std::shared_ptr<StorageApiChainBmFeedHandler::Context> context); + ~MyStorageChainBuilder() override; + void add(std::unique_ptr<StorageLink> link) override; +}; + +MyStorageChainBuilder::MyStorageChainBuilder(std::shared_ptr<StorageApiChainBmFeedHandler::Context> context) + : storage::StorageChainBuilder(), + _context(std::move(context)) +{ +} + +MyStorageChainBuilder::~MyStorageChainBuilder() = default; + +void +MyStorageChainBuilder::add(std::unique_ptr<StorageLink> link) +{ + vespalib::string name = link->getName(); + Parent::add(std::move(link)); + if (name == "Communication manager") { + auto my_link = std::make_unique<BmStorageLink>(); + _context->bm_link = my_link.get(); + Parent::add(std::move(my_link)); + } +} + +StorageApiChainBmFeedHandler::StorageApiChainBmFeedHandler(std::shared_ptr<Context> context) + : IBmFeedHandler(), + _context(std::move(context)) +{ + auto cmd = make_set_cluster_state_cmd(); + PendingTracker tracker(1); + send_msg(std::move(cmd), tracker); + tracker.drain(); +} + +StorageApiChainBmFeedHandler::~StorageApiChainBmFeedHandler() = default; + +void +StorageApiChainBmFeedHandler::send_msg(std::shared_ptr<storage::api::StorageCommand> cmd, PendingTracker& pending_tracker) +{ + cmd->setSourceIndex(0); + auto bm_link = _context->bm_link; + bm_link->retain(cmd->getMsgId(), pending_tracker); + bm_link->sendDown(std::move(cmd)); +} + +void +StorageApiChainBmFeedHandler::put(const document::Bucket& bucket, std::unique_ptr<Document> document, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique<storage::api::PutCommand>(bucket, std::move(document), timestamp); + send_msg(std::move(cmd), tracker); +} + +void +StorageApiChainBmFeedHandler::update(const document::Bucket& bucket, std::unique_ptr<DocumentUpdate> document_update, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique<storage::api::UpdateCommand>(bucket, std::move(document_update), timestamp); + send_msg(std::move(cmd), tracker); +} + +void +StorageApiChainBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique<storage::api::RemoveCommand>(bucket, document_id, timestamp); + send_msg(std::move(cmd), tracker); +} + +std::shared_ptr<StorageApiChainBmFeedHandler::Context> +StorageApiChainBmFeedHandler::get_context() +{ + return std::make_shared<Context>(); +} + +std::unique_ptr<storage::IStorageChainBuilder> +StorageApiChainBmFeedHandler::get_storage_chain_builder(std::shared_ptr<Context> context) +{ + return std::make_unique<MyStorageChainBuilder>(std::move(context)); +} + +} diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h new file mode 100644 index 00000000000..521deddd19e --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_chain_bm_feed_handler.h @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_bm_feed_handler.h" + +namespace storage { class IStorageChainBuilder; } +namespace storage::api { class StorageCommand; } + +namespace feedbm { + +/* + * Benchmark feed handler for feed to service layer using storage api protocol + * directly on the storage chain. + */ +class StorageApiChainBmFeedHandler : public IBmFeedHandler +{ +public: + struct Context; +private: + std::shared_ptr<Context> _context; + void send_msg(std::shared_ptr<storage::api::StorageCommand> cmd, PendingTracker& tracker); +public: + StorageApiChainBmFeedHandler(std::shared_ptr<Context> context); + ~StorageApiChainBmFeedHandler(); + void put(const document::Bucket& bucket, std::unique_ptr<document::Document> document, uint64_t timestamp, PendingTracker& tracker) override; + void update(const document::Bucket& bucket, std::unique_ptr<document::DocumentUpdate> document_update, uint64_t timestamp, PendingTracker& tracker) override; + void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) override; + + static std::shared_ptr<Context> get_context(); + static std::unique_ptr<storage::IStorageChainBuilder> get_storage_chain_builder(std::shared_ptr<Context> context); +}; + +} diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp new file mode 100644 index 00000000000..c8d73444652 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.cpp @@ -0,0 +1,146 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_api_rpc_bm_feed_handler.h" +#include "pending_tracker.h" +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/storageserver/message_dispatcher.h> +#include <vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h> +#include <vespa/storage/storageserver/rpc/message_codec_provider.h> +#include <vespa/storage/storageserver/rpc/shared_rpc_resources.h> +#include <vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.h> +#include <vespa/storage/storageserver/rpc/storage_api_rpc_service.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/slobrok/sbmirror.h> +#include <cassert> + +using document::Document; +using document::DocumentId; +using document::DocumentUpdate; +using document::DocumentTypeRepo; +using storage::api::StorageMessageAddress; +using storage::rpc::SharedRpcResources; + +namespace feedbm { + +namespace { + +FRT_RPCRequest * +make_set_cluster_state_request() { + storage::lib::ClusterStateBundle bundle(storage::lib::ClusterState("version:2 distributor:1 storage:1")); + storage::rpc::SlimeClusterStateBundleCodec codec; + auto encoded_bundle = codec.encode(bundle); + auto *req = new FRT_RPCRequest(); + auto* params = req->GetParams(); + params->AddInt8(static_cast<uint8_t>(encoded_bundle._compression_type)); + params->AddInt32(encoded_bundle._uncompressed_length); + params->AddData(std::move(*encoded_bundle._buffer)); + req->SetMethodName("setdistributionstates"); + return req; +} + +void +set_cluster_up(SharedRpcResources &shared_rpc_resources, storage::api::StorageMessageAddress &storage_address) { + auto req = make_set_cluster_state_request(); + auto target_resolver = std::make_unique<storage::rpc::CachingRpcTargetResolver>(shared_rpc_resources.slobrok_mirror(), shared_rpc_resources.target_factory()); + auto target = target_resolver->resolve_rpc_target(storage_address); + target->_target->get()->InvokeSync(req, 10.0); // 10 seconds timeout + assert(!req->IsError()); + req->SubRef(); +} + +} + +class StorageApiRpcBmFeedHandler::MyMessageDispatcher : public storage::MessageDispatcher +{ + std::mutex _mutex; + vespalib::hash_map<uint64_t, PendingTracker *> _pending; +public: + MyMessageDispatcher() + : storage::MessageDispatcher(), + _mutex(), + _pending() + { + } + ~MyMessageDispatcher() override; + void dispatch_sync(std::shared_ptr<storage::api::StorageMessage> msg) override { + release(msg->getMsgId()); + } + void dispatch_async(std::shared_ptr<storage::api::StorageMessage> msg) override { + release(msg->getMsgId()); + } + void retain(uint64_t msg_id, PendingTracker &tracker) { + tracker.retain(); + std::lock_guard lock(_mutex); + _pending.insert(std::make_pair(msg_id, &tracker)); + } + void release(uint64_t msg_id) { + PendingTracker *tracker = nullptr; + { + std::lock_guard lock(_mutex); + auto itr = _pending.find(msg_id); + assert(itr != _pending.end()); + tracker = itr->second; + _pending.erase(itr); + } + tracker->release(); + } +}; + +StorageApiRpcBmFeedHandler::MyMessageDispatcher::~MyMessageDispatcher() +{ + std::lock_guard lock(_mutex); + assert(_pending.empty()); +} + +StorageApiRpcBmFeedHandler::StorageApiRpcBmFeedHandler(SharedRpcResources& shared_rpc_resources_in, std::shared_ptr<const DocumentTypeRepo> repo) + : IBmFeedHandler(), + _storage_address(std::make_unique<StorageMessageAddress>("storage", storage::lib::NodeType::STORAGE, 0)), + _shared_rpc_resources(shared_rpc_resources_in), + _message_dispatcher(std::make_unique<MyMessageDispatcher>()), + _message_codec_provider(std::make_unique<storage::rpc::MessageCodecProvider>(repo, std::make_shared<documentapi::LoadTypeSet>())), + _rpc_client(std::make_unique<storage::rpc::StorageApiRpcService>(*_message_dispatcher, _shared_rpc_resources, *_message_codec_provider, storage::rpc::StorageApiRpcService::Params())) +{ + set_cluster_up(_shared_rpc_resources, *_storage_address); +} + +StorageApiRpcBmFeedHandler::~StorageApiRpcBmFeedHandler() = default; + +void +StorageApiRpcBmFeedHandler::send_rpc(std::shared_ptr<storage::api::StorageCommand> cmd, PendingTracker& pending_tracker) +{ + cmd->setSourceIndex(0); + cmd->setAddress(*_storage_address); + _message_dispatcher->retain(cmd->getMsgId(), pending_tracker); + _rpc_client->send_rpc_v1_request(std::move(cmd)); +} + +void +StorageApiRpcBmFeedHandler::put(const document::Bucket& bucket, std::unique_ptr<Document> document, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique<storage::api::PutCommand>(bucket, std::move(document), timestamp); + send_rpc(std::move(cmd), tracker); +} + +void +StorageApiRpcBmFeedHandler::update(const document::Bucket& bucket, std::unique_ptr<DocumentUpdate> document_update, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique<storage::api::UpdateCommand>(bucket, std::move(document_update), timestamp); + send_rpc(std::move(cmd), tracker); +} + +void +StorageApiRpcBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) +{ + auto cmd = std::make_unique<storage::api::RemoveCommand>(bucket, document_id, timestamp); + send_rpc(std::move(cmd), tracker); +} + +} diff --git a/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.h b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.h new file mode 100644 index 00000000000..0fe92350eb2 --- /dev/null +++ b/searchcore/src/apps/vespa-feed-bm/storage_api_rpc_bm_feed_handler.h @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_bm_feed_handler.h" + +namespace document { class DocumentTypeRepo; } +namespace storage::api { +class StorageMessageAddress; +class StorageCommand; +} + +namespace storage::rpc { +class MessageCodecProvider; +class SharedRpcResources; +class StorageApiRpcService; +} + +namespace feedbm { + +/* + * Benchmark feed handler for feed to service layer using storage api protocol + * over rpc. + */ +class StorageApiRpcBmFeedHandler : public IBmFeedHandler +{ + class MyMessageDispatcher; + std::unique_ptr<storage::api::StorageMessageAddress> _storage_address; + storage::rpc::SharedRpcResources& _shared_rpc_resources; + std::unique_ptr<MyMessageDispatcher> _message_dispatcher; + std::unique_ptr<storage::rpc::MessageCodecProvider> _message_codec_provider; + std::unique_ptr<storage::rpc::StorageApiRpcService> _rpc_client; + + void send_rpc(std::shared_ptr<storage::api::StorageCommand> cmd, PendingTracker& tracker); +public: + StorageApiRpcBmFeedHandler(storage::rpc::SharedRpcResources& shared_rpc_resources_in, std::shared_ptr<const document::DocumentTypeRepo> repo); + ~StorageApiRpcBmFeedHandler(); + void put(const document::Bucket& bucket, std::unique_ptr<document::Document> document, uint64_t timestamp, PendingTracker& tracker) override; + void update(const document::Bucket& bucket, std::unique_ptr<document::DocumentUpdate> document_update, uint64_t timestamp, PendingTracker& tracker) override; + void remove(const document::Bucket& bucket, const document::DocumentId& document_id, uint64_t timestamp, PendingTracker& tracker) override; +}; + +} diff --git a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index ecfa2a07cef..73d741c3fee 100644 --- a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -1,7 +1,10 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "pending_tracker.h" +#include "spi_bm_feed_handler.h" +#include "storage_api_rpc_bm_feed_handler.h" +#include "storage_api_chain_bm_feed_handler.h" #include <vespa/vespalib/testkit/testapp.h> - #include <tests/proton/common/dummydbowner.h> #include <vespa/config-imported-fields.h> #include <vespa/config-rank-profiles.h> @@ -11,6 +14,7 @@ #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/document_type_repo_factory.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/documentupdate.h> @@ -37,11 +41,32 @@ #include <vespa/config-summary.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/fastos/app.h> +#include <vespa/storage/bucketdb/config-stor-bucket-init.h> +#include <vespa/storage/config/config-stor-bouncer.h> +#include <vespa/storage/config/config-stor-communicationmanager.h> +#include <vespa/storage/config/config-stor-opslogger.h> +#include <vespa/storage/config/config-stor-prioritymapping.h> +#include <vespa/storage/config/config-stor-server.h> +#include <vespa/storage/config/config-stor-status.h> +#include <vespa/storage/visiting/config-stor-visitor.h> +#include <vespa/config-load-type.h> +#include <vespa/config-persistence.h> +#include <vespa/config-stor-distribution.h> +#include <vespa/config-stor-filestor.h> +#include <vespa/config-upgrading.h> +#include <vespa/config-slobroks.h> +#include <vespa/metrics/config-metricsmanager.h> +#include <vespa/storageserver/app/servicelayerprocess.h> +#include <vespa/storage/common/i_storage_chain_builder.h> +#include <vespa/storage/storageserver/storagenode.h> +#include <vespa/messagebus/config-messagebus.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/storage/storageserver/rpc/shared_rpc_resources.h> #include <getopt.h> #include <iostream> #include <vespa/log/log.h> -LOG_SETUP("vespa-spi-feed-bm"); +LOG_SETUP("vespa-feed-bm"); using namespace config; using namespace proton; @@ -51,7 +76,28 @@ using namespace vespa::config::search::summary; using namespace vespa::config::search; using namespace std::chrono_literals; using vespa::config::content::core::BucketspacesConfig; - +using vespa::config::content::core::BucketspacesConfigBuilder; +using vespa::config::content::StorDistributionConfigBuilder; +using vespa::config::content::StorFilestorConfigBuilder; +using vespa::config::content::PersistenceConfigBuilder; +using vespa::config::content::core::StorBouncerConfigBuilder; +using vespa::config::content::core::StorCommunicationmanagerConfigBuilder; +using vespa::config::content::core::StorBucketInitConfigBuilder; +using vespa::config::content::core::StorOpsloggerConfigBuilder; +using vespa::config::content::core::StorPrioritymappingConfigBuilder; +using vespa::config::content::LoadTypeConfigBuilder; +using vespa::config::content::UpgradingConfigBuilder; +using vespa::config::content::core::StorServerConfigBuilder; +using vespa::config::content::core::StorStatusConfigBuilder; +using vespa::config::content::core::StorVisitorConfigBuilder; +using metrics::MetricsmanagerConfigBuilder; +using cloud::config::SlobroksConfigBuilder; +using messagebus::MessagebusConfigBuilder; + +using config::ConfigContext; +using config::ConfigUri; +using config::ConfigSet; +using config::IConfigContext; using document::AssignValueUpdate; using document::BucketId; using document::BucketSpace; @@ -59,7 +105,9 @@ using document::Document; using document::DocumentId; using document::DocumentType; using document::DocumentTypeRepo; +using document::DocumentTypeRepoFactory; using document::DocumenttypesConfig; +using document::DocumenttypesConfigBuilder; using document::DocumentUpdate; using document::Field; using document::FieldUpdate; @@ -70,20 +118,18 @@ using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::SchemaBuilder; using search::transactionlog::TransLogServer; -using storage::spi::Bucket; -using storage::spi::PartitionId; +using storage::rpc::SharedRpcResources; using storage::spi::PersistenceProvider; -using storage::spi::Priority; -using storage::spi::Timestamp; -using storage::spi::Trace; using vespalib::makeLambdaTask; +using feedbm::IBmFeedHandler; +using feedbm::SpiBmFeedHandler; +using feedbm::StorageApiRpcBmFeedHandler; +using feedbm::StorageApiChainBmFeedHandler; using DocumentDBMap = std::map<DocTypeName, std::shared_ptr<DocumentDB>>; namespace { -storage::spi::LoadType default_load_type(0, "default"); - vespalib::string base_dir = "testdb"; std::shared_ptr<DocumenttypesConfig> make_document_type() { @@ -144,84 +190,6 @@ struct MyResourceWriteFilter : public IResourceWriteFilter State getAcceptState() const override { return IResourceWriteFilter::State(); } }; -class MyPendingTracker { - uint32_t _pending; - uint32_t _limit; - std::mutex _mutex; - std::condition_variable _cond; - -public: - MyPendingTracker(uint32_t limit) - : _pending(0u), - _limit(limit), - _mutex(), - _cond() - { - } - - ~MyPendingTracker() - { - drain(); - } - - void release() { - std::unique_lock<std::mutex> guard(_mutex); - --_pending; - if (_pending < _limit) { - _cond.notify_all(); - } - //LOG(info, "release, pending is now %u", _pending); - } - void retain() { - std::unique_lock<std::mutex> guard(_mutex); - while (_pending >= _limit) { - _cond.wait(guard); - } - ++_pending; - //LOG(info, "retain, pending is now %u", _pending); - } - - void drain() { - std::unique_lock<std::mutex> guard(_mutex); - while (_pending > 0) { - _cond.wait(guard); - } - } -}; - -class MyOperationComplete : public storage::spi::OperationComplete -{ - MyPendingTracker& _tracker; -public: - MyOperationComplete(MyPendingTracker &tracker); - ~MyOperationComplete(); - void onComplete(std::unique_ptr<storage::spi::Result> result) override; - void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; -}; - -MyOperationComplete::MyOperationComplete(MyPendingTracker& tracker) - : _tracker(tracker) -{ - _tracker.retain(); -} - -MyOperationComplete::~MyOperationComplete() -{ - _tracker.release(); -} - -void -MyOperationComplete::onComplete(std::unique_ptr<storage::spi::Result> result) -{ - (void) result; -} - -void -MyOperationComplete::addResultHandler(const storage::spi::ResultHandler * resultHandler) -{ - (void) resultHandler; -} - class BMRange { uint32_t _start; @@ -242,6 +210,9 @@ class BMParams { uint32_t _put_passes; uint32_t _update_passes; uint32_t _remove_passes; + uint32_t _rpc_network_threads; + bool _enable_service_layer; + bool _use_storage_chain; uint32_t get_start(uint32_t thread_id) const { return (_documents / _threads) * thread_id + std::min(thread_id, _documents % _threads); } @@ -251,7 +222,10 @@ public: _threads(32), _put_passes(2), _update_passes(1), - _remove_passes(2) + _remove_passes(2), + _rpc_network_threads(1), + _enable_service_layer(false), + _use_storage_chain(false) { } BMRange get_range(uint32_t thread_id) const { @@ -262,11 +236,17 @@ public: uint32_t get_put_passes() const { return _put_passes; } uint32_t get_update_passes() const { return _update_passes; } uint32_t get_remove_passes() const { return _remove_passes; } + uint32_t get_rpc_network_threads() const { return _rpc_network_threads; } + bool get_enable_service_layer() const { return _enable_service_layer; } + bool get_use_storage_chain() const { return _use_storage_chain; } void set_documents(uint32_t documents_in) { _documents = documents_in; } void set_threads(uint32_t threads_in) { _threads = threads_in; } void set_put_passes(uint32_t put_passes_in) { _put_passes = put_passes_in; } void set_update_passes(uint32_t update_passes_in) { _update_passes = update_passes_in; } void set_remove_passes(uint32_t remove_passes_in) { _remove_passes = remove_passes_in; } + void set_rpc_network_threads(uint32_t threads_in) { _rpc_network_threads = threads_in; } + void set_enable_service_layer(bool enable_service_layer_in) { _enable_service_layer = enable_service_layer_in; } + void set_use_storage_chain(bool use_storage_chain_in) { _use_storage_chain = use_storage_chain_in; } bool check() const; }; @@ -289,11 +269,177 @@ BMParams::check() const std::cerr << "Put passes too low: " << _put_passes << std::endl; return false; } + if (_rpc_network_threads < 1) { + std::cerr << "Too few rpc network threads: " << _rpc_network_threads << std::endl; + return false; + } return true; } +class MyServiceLayerProcess : public storage::ServiceLayerProcess { + PersistenceProvider& _provider; + +public: + MyServiceLayerProcess(const config::ConfigUri & configUri, + PersistenceProvider &provider, + std::unique_ptr<storage::IStorageChainBuilder> chain_builder); + ~MyServiceLayerProcess() override { shutdown(); } + + void shutdown() override; + void setupProvider() override; + PersistenceProvider& getProvider() override; +}; + +MyServiceLayerProcess::MyServiceLayerProcess(const config::ConfigUri & configUri, + PersistenceProvider &provider, + std::unique_ptr<storage::IStorageChainBuilder> chain_builder) + : ServiceLayerProcess(configUri), + _provider(provider) +{ + if (chain_builder) { + set_storage_chain_builder(std::move(chain_builder)); + } +} + +void +MyServiceLayerProcess::shutdown() +{ + ServiceLayerProcess::shutdown(); } +void +MyServiceLayerProcess::setupProvider() +{ +} + +PersistenceProvider& +MyServiceLayerProcess::getProvider() +{ + return _provider; +} + +struct MyStorageConfig +{ + vespalib::string config_id; + DocumenttypesConfigBuilder documenttypes; + PersistenceConfigBuilder persistence; + StorDistributionConfigBuilder stor_distribution; + StorFilestorConfigBuilder stor_filestor; + StorBouncerConfigBuilder stor_bouncer; + StorCommunicationmanagerConfigBuilder stor_communicationmanager; + StorBucketInitConfigBuilder stor_bucket_init; + StorOpsloggerConfigBuilder stor_opslogger; + StorPrioritymappingConfigBuilder stor_prioritymapping; + UpgradingConfigBuilder upgrading; + StorServerConfigBuilder stor_server; + StorStatusConfigBuilder stor_status; + StorVisitorConfigBuilder stor_visitor; + BucketspacesConfigBuilder bucketspaces; + LoadTypeConfigBuilder load_type; + MetricsmanagerConfigBuilder metricsmanager; + SlobroksConfigBuilder slobroks; + MessagebusConfigBuilder messagebus; + + MyStorageConfig(const vespalib::string& config_id_in, const DocumenttypesConfig& documenttypes_in, int slobrok_port, int status_port, uint32_t rpc_network_threads) + : config_id(config_id_in), + documenttypes(documenttypes_in), + persistence(), + stor_distribution(), + stor_filestor(), + stor_bouncer(), + stor_communicationmanager(), + stor_bucket_init(), + stor_opslogger(), + stor_prioritymapping(), + upgrading(), + stor_server(), + stor_status(), + stor_visitor(), + bucketspaces(), + load_type(), + metricsmanager(), + slobroks(), + messagebus() + { + { + auto &dc = stor_distribution; + { + StorDistributionConfigBuilder::Group group; + { + StorDistributionConfigBuilder::Group::Nodes node; + node.index = 0; + group.nodes.push_back(std::move(node)); + } + group.index = "invalid"; + group.name = "invalid"; + group.capacity = 1.0; + group.partitions = ""; + dc.group.push_back(std::move(group)); + } + dc.redundancy = 1; + dc.readyCopies = 1; + } + stor_server.rootFolder = "storage"; + { + SlobroksConfigBuilder::Slobrok slobrok; + slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); + slobroks.slobrok.push_back(std::move(slobrok)); + } + stor_communicationmanager.useDirectStorageapiRpc = true; + stor_communicationmanager.rpc.numNetworkThreads = rpc_network_threads; + stor_status.httpport = status_port; + } + + ~MyStorageConfig(); + + void add_builders(ConfigSet &set) { + set.addBuilder(config_id, &documenttypes); + set.addBuilder(config_id, &persistence); + set.addBuilder(config_id, &stor_distribution); + set.addBuilder(config_id, &stor_filestor); + set.addBuilder(config_id, &stor_bouncer); + set.addBuilder(config_id, &stor_communicationmanager); + set.addBuilder(config_id, &stor_bucket_init); + set.addBuilder(config_id, &stor_opslogger); + set.addBuilder(config_id, &stor_prioritymapping); + set.addBuilder(config_id, &upgrading); + set.addBuilder(config_id, &stor_server); + set.addBuilder(config_id, &stor_status); + set.addBuilder(config_id, &stor_visitor); + set.addBuilder(config_id, &bucketspaces); + set.addBuilder(config_id, &load_type); + set.addBuilder(config_id, &metricsmanager); + set.addBuilder(config_id, &slobroks); + set.addBuilder(config_id, &messagebus); + } +}; + +MyStorageConfig::~MyStorageConfig() = default; + +struct MyRpcClientConfig { + vespalib::string config_id; + SlobroksConfigBuilder slobroks; + + MyRpcClientConfig(const vespalib::string &config_id_in, int slobrok_port) + : config_id(config_id_in), + slobroks() + { + { + SlobroksConfigBuilder::Slobrok slobrok; + slobrok.connectionspec = vespalib::make_string("tcp/localhost:%d", slobrok_port); + slobroks.slobrok.push_back(std::move(slobrok)); + } + } + ~MyRpcClientConfig(); + + void add_builders(ConfigSet &set) { + set.addBuilder(config_id, &slobroks); + } +}; + +MyRpcClientConfig::~MyRpcClientConfig() = default; + +} struct PersistenceProviderFixture { std::shared_ptr<DocumenttypesConfig> _document_types; @@ -305,6 +451,9 @@ struct PersistenceProviderFixture { vespalib::string _base_dir; DummyFileHeaderContext _file_header_context; int _tls_listen_port; + int _slobrok_port; + int _status_port; + int _rpc_client_port; TransLogServer _tls; vespalib::string _tls_spec; matching::QueryLimiter _query_limiter; @@ -318,24 +467,33 @@ struct PersistenceProviderFixture { MyPersistenceEngineOwner _persistence_owner; MyResourceWriteFilter _write_filter; std::shared_ptr<PersistenceEngine> _persistence_engine; - storage::spi::Context _context; uint32_t _bucket_bits; - - PersistenceProviderFixture(); + MyStorageConfig _service_layer_config; + MyRpcClientConfig _rpc_client_config; + ConfigSet _config_set; + std::shared_ptr<IConfigContext> _config_context; + std::unique_ptr<IBmFeedHandler> _feed_handler; + std::unique_ptr<mbus::Slobrok> _slobrok; + std::unique_ptr<MyServiceLayerProcess> _service_layer; + std::unique_ptr<SharedRpcResources> _rpc_client_shared_rpc_resources; + + PersistenceProviderFixture(const BMParams& params); ~PersistenceProviderFixture(); void create_document_db(); uint32_t num_buckets() const { return (1u << _bucket_bits); } BucketId make_bucket_id(uint32_t i) const { return BucketId(_bucket_bits, i & (num_buckets() - 1)); } - Bucket make_bucket(uint32_t i) const { return Bucket(document::Bucket(_bucket_space, BucketId(_bucket_bits, i & (num_buckets() - 1))), PartitionId(0)); } + document::Bucket make_bucket(uint32_t i) const { return document::Bucket(_bucket_space, BucketId(_bucket_bits, i & (num_buckets() - 1))); } DocumentId make_document_id(uint32_t i) const; std::unique_ptr<Document> make_document(uint32_t i) const; std::unique_ptr<DocumentUpdate> make_document_update(uint32_t i) const; void create_buckets(); + void start_service_layer(bool use_storage_chain); + void shutdown_service_layer(); }; -PersistenceProviderFixture::PersistenceProviderFixture() +PersistenceProviderFixture::PersistenceProviderFixture(const BMParams& params) : _document_types(make_document_type()), - _repo(std::make_shared<DocumentTypeRepo>(*_document_types)), + _repo(DocumentTypeRepoFactory::make(*_document_types)), _doc_type_name("test"), _document_type(_repo->getDocumentType(_doc_type_name.getName())), _field(_document_type->getField("int")), @@ -343,6 +501,9 @@ PersistenceProviderFixture::PersistenceProviderFixture() _base_dir(base_dir), _file_header_context(), _tls_listen_port(9017), + _slobrok_port(9018), + _status_port(9019), + _rpc_client_port(9020), _tls("tls", _tls_listen_port, _base_dir, _file_header_context), _tls_spec(vespalib::make_string("tcp/localhost:%d", _tls_listen_port)), _query_limiter(), @@ -356,13 +517,23 @@ PersistenceProviderFixture::PersistenceProviderFixture() _persistence_owner(), _write_filter(), _persistence_engine(), - _context(default_load_type, Priority(0), Trace::TraceLevel(0)), - _bucket_bits(16) + _bucket_bits(16), + _service_layer_config("bm-servicelayer", *_document_types, _slobrok_port, _status_port, params.get_rpc_network_threads()), + _rpc_client_config("bm-rpc-client", _slobrok_port), + _config_set(), + _config_context(std::make_shared<ConfigContext>(_config_set)), + _feed_handler(), + _slobrok(), + _service_layer(), + _rpc_client_shared_rpc_resources() { create_document_db(); _persistence_engine = std::make_unique<PersistenceEngine>(_persistence_owner, _write_filter, -1, false); auto proxy = std::make_shared<PersistenceHandlerProxy>(_document_db); _persistence_engine->putHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name, proxy); + _service_layer_config.add_builders(_config_set); + _rpc_client_config.add_builders(_config_set); + _feed_handler = std::make_unique<SpiBmFeedHandler>(*_persistence_engine); } PersistenceProviderFixture::~PersistenceProviderFixture() @@ -448,9 +619,59 @@ PersistenceProviderFixture::make_document_update(uint32_t i) const void PersistenceProviderFixture::create_buckets() { - auto &provider = *_persistence_engine; + SpiBmFeedHandler feed_handler(*_persistence_engine); for (unsigned int i = 0; i < num_buckets(); ++i) { - provider.createBucket(make_bucket(i), _context); + feed_handler.create_bucket(make_bucket(i)); + } +} + +void +PersistenceProviderFixture::start_service_layer(bool use_storage_chain) +{ + LOG(info, "start slobrok"); + _slobrok = std::make_unique<mbus::Slobrok>(_slobrok_port); + LOG(info, "start service layer"); + config::ConfigUri config_uri("bm-servicelayer", _config_context); + std::unique_ptr<storage::IStorageChainBuilder> chain_builder; + std::shared_ptr<StorageApiChainBmFeedHandler::Context> context; + if (use_storage_chain) { + context = StorageApiChainBmFeedHandler::get_context(); + chain_builder = StorageApiChainBmFeedHandler::get_storage_chain_builder(context); + } + _service_layer = std::make_unique<MyServiceLayerProcess>(config_uri, + *_persistence_engine, + std::move(chain_builder)); + _service_layer->setupConfig(100ms); + _service_layer->createNode(); + _service_layer->getNode().waitUntilInitialized(); + LOG(info, "start rpc client shared resources"); + config::ConfigUri client_config_uri("bm-rpc-client", _config_context); + _rpc_client_shared_rpc_resources = std::make_unique<SharedRpcResources>(client_config_uri, _rpc_client_port, 100); + _rpc_client_shared_rpc_resources->start_server_and_register_slobrok("bm-rpc-client"); + if (use_storage_chain) { + _feed_handler = std::make_unique<StorageApiChainBmFeedHandler>(std::move(context)); + } else { + _feed_handler = std::make_unique<StorageApiRpcBmFeedHandler>(*_rpc_client_shared_rpc_resources, _repo); + } +} + +void +PersistenceProviderFixture::shutdown_service_layer() +{ + _feed_handler.reset(); + if (_rpc_client_shared_rpc_resources) { + LOG(info, "stop rpc client shared resources"); + _rpc_client_shared_rpc_resources->shutdown(); + _rpc_client_shared_rpc_resources.reset(); + } + if (_service_layer) { + LOG(info, "stop service layer"); + _service_layer->getNode().requestShutdown("controlled shutdown"); + _service_layer->shutdown(); + } + if (_slobrok) { + LOG(info, "stop slobrok"); + _slobrok.reset(); } } @@ -490,18 +711,16 @@ void put_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) { LOG(debug, "put_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; + feedbm::PendingTracker pending_tracker(100); auto &repo = *f._repo; vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); BucketId bucket_id; auto bucket_space = f._bucket_space; for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + document::Bucket bucket(bucket_space, bucket_id); auto document = std::make_unique<Document>(repo, is); - provider.putAsync(bucket, Timestamp(time_bias + i), std::move(document), context, std::make_unique<MyOperationComplete>(pending_tracker)); + f._feed_handler->put(bucket, std::move(document), time_bias + i, pending_tracker); } assert(is.empty()); pending_tracker.drain(); @@ -541,18 +760,16 @@ void update_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) { LOG(debug, "update_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; + feedbm::PendingTracker pending_tracker(100); auto &repo = *f._repo; vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); BucketId bucket_id; auto bucket_space = f._bucket_space; for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + document::Bucket bucket(bucket_space, bucket_id); auto document_update = DocumentUpdate::createHEAD(repo, is); - provider.updateAsync(bucket, Timestamp(time_bias + i), std::move(document_update), context, std::make_unique<MyOperationComplete>(pending_tracker)); + f._feed_handler->update(bucket, std::move(document_update), time_bias + i, pending_tracker); } assert(is.empty()); pending_tracker.drain(); @@ -593,17 +810,15 @@ void remove_async_task(PersistenceProviderFixture &f, BMRange range, const vespalib::nbostream &serialized_feed, int64_t time_bias) { LOG(debug, "remove_async_task([%u..%u))", range.get_start(), range.get_end()); - MyPendingTracker pending_tracker(100); - auto &provider = *f._persistence_engine; - auto &context = f._context; + feedbm::PendingTracker pending_tracker(100); vespalib::nbostream is(serialized_feed.data(), serialized_feed.size()); BucketId bucket_id; auto bucket_space = f._bucket_space; for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { is >> bucket_id; - Bucket bucket(document::Bucket(bucket_space, bucket_id), PartitionId(0)); + document::Bucket bucket(bucket_space, bucket_id); DocumentId document_id(is); - provider.removeAsync(bucket, Timestamp(time_bias + i), document_id, context, std::make_unique<MyOperationComplete>(pending_tracker)); + f._feed_handler->remove(bucket, document_id, time_bias + i, pending_tracker); } assert(is.empty()); pending_tracker.drain(); @@ -629,12 +844,15 @@ run_remove_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecu void benchmark_async_spi(const BMParams &bm_params) { vespalib::rmdir(base_dir, true); - PersistenceProviderFixture f; + PersistenceProviderFixture f(bm_params); auto &provider = *f._persistence_engine; LOG(info, "start initialize"); provider.initialize(); LOG(info, "create %u buckets", f.num_buckets()); f.create_buckets(); + if (bm_params.get_enable_service_layer()) { + f.start_service_layer(bm_params.get_use_storage_chain()); + } vespalib::ThreadStackExecutor executor(bm_params.get_threads(), 128 * 1024); auto put_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_put_feed(f, range); }, "put"); auto update_feed = make_feed(executor, bm_params, [&f](BMRange range) { return make_update_feed(f, range); }, "update"); @@ -649,6 +867,7 @@ void benchmark_async_spi(const BMParams &bm_params) for (uint32_t pass = 0; pass < bm_params.get_remove_passes(); ++pass) { run_remove_async_tasks(f, executor, pass, time_bias, remove_feed, bm_params); } + f.shutdown_service_layer(); } class App : public FastOS_Application @@ -673,16 +892,19 @@ void App::usage() { std::cerr << - "vespa-spi-feed-bm version 0.0\n" + "vespa-feed-bm version 0.0\n" "\n" "USAGE:\n"; std::cerr << - "vespa-spi-feed-bm\n" + "vespa-feed-bm\n" "[--threads threads]\n" "[--documents documents]\n" "[--put-passes put-passes]\n" "[--update-passes update-passes]\n" - "[--remove-passes remove-passes]" << std::endl; + "[--remove-passes remove-passes]\n" + "[--rpc-network-threads threads]\n" + "[--enable-service-layer]\n" + "[--use-storage-chain]" << std::endl; } bool @@ -696,14 +918,20 @@ App::get_options() { "documents", 1, nullptr, 0 }, { "put-passes", 1, nullptr, 0 }, { "update-passes", 1, nullptr, 0 }, - { "remove-passes", 1, nullptr, 0 } + { "remove-passes", 1, nullptr, 0 }, + { "rpc-network-threads", 1, nullptr, 0 }, + { "enable-service-layer", 0, nullptr, 0 }, + { "use-storage-chain", 0, nullptr, 0 } }; enum longopts_enum { LONGOPT_THREADS, LONGOPT_DOCUMENTS, LONGOPT_PUT_PASSES, LONGOPT_UPDATE_PASSES, - LONGOPT_REMOVE_PASSES + LONGOPT_REMOVE_PASSES, + LONGOPT_RPC_NETWORK_THREADS, + LONGOPT_ENABLE_SERVICE_LAYER, + LONGOPT_USE_STORAGE_CHAIN }; int opt_index = 1; resetOptIndex(opt_index); @@ -726,6 +954,15 @@ App::get_options() case LONGOPT_REMOVE_PASSES: _bm_params.set_remove_passes(atoi(opt_argument)); break; + case LONGOPT_RPC_NETWORK_THREADS: + _bm_params.set_rpc_network_threads(atoi(opt_argument)); + break; + case LONGOPT_ENABLE_SERVICE_LAYER: + _bm_params.set_enable_service_layer(true); + break; + case LONGOPT_USE_STORAGE_CHAIN: + _bm_params.set_use_storage_chain(true); + break; default: return false; } @@ -751,7 +988,7 @@ App::Main() int main(int argc, char* argv[]) { - DummyFileHeaderContext::setCreator("vespa-spi-feed-bm"); + DummyFileHeaderContext::setCreator("vespa-feed-bm"); App app; auto exit_value = app.Entry(argc, argv); vespalib::rmdir(base_dir, true); diff --git a/searchcore/src/apps/vespa-spi-feed-bm/.gitignore b/searchcore/src/apps/vespa-spi-feed-bm/.gitignore deleted file mode 100644 index 02fff2fe280..00000000000 --- a/searchcore/src/apps/vespa-spi-feed-bm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vespa-spi-feed-bm diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 41de6827244..8711a21e5e6 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -192,6 +192,11 @@ public: void assertExecuteHistory(std::vector<uint32_t> expExecuteHistory) { EXPECT_EQ(expExecuteHistory, _attributeFieldWriter->getExecuteHistory()); } + SerialNum test_force_commit(AttributeVector &attr, SerialNum serialNum) { + commit(serialNum); + _attributeFieldWriter->sync(); + return attr.getStatus().getLastSyncToken(); + } }; AttributeWriterTest::~AttributeWriterTest() = default; @@ -975,6 +980,15 @@ TEST_F(AttributeWriterTest, forceCommit_clears_search_cache_in_imported_attribut EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } +TEST_F(AttributeWriterTest, ignores_force_commit_serial_not_greater_than_create_serial) +{ + auto a1 = addAttribute("a1"); + a1->setCreateSerialNum(100); + EXPECT_EQ(0u, test_force_commit(*a1, 50u)); + EXPECT_EQ(0u, test_force_commit(*a1, 100u)); + EXPECT_EQ(150u, test_force_commit(*a1, 150u)); +} + class StructWriterTestBase : public AttributeWriterTest { public: DocumentType _type; diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index ad0ce0b26c4..72592cca681 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -434,6 +434,7 @@ struct MyTlsWriter : TlsWriter { MyTlsWriter() : store_count(0), erase_count(0), erase_return(true) {} void appendOperation(const FeedOperation &, DoneCallback) override { ++store_count; } + CommitResult startCommit(DoneCallback) override { return CommitResult(); } bool erase(SerialNum) override { ++erase_count; return erase_return; } SerialNum sync(SerialNum syncTo) override { diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 31882061b1c..04444647b5d 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -172,6 +172,9 @@ struct MyStorer : public IOperationStorer { ++_compactCnt; } } + CommitResult startCommit(DoneCallback) override { + return CommitResult(); + } }; struct MyFrozenBucketHandler : public IFrozenBucketHandler { diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 8df62705cb3..f033dfd50a8 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -235,6 +235,9 @@ public: // Implements IOperationStorer void appendOperation(const FeedOperation &op, DoneCallback) override; + CommitResult startCommit(DoneCallback) override { + return CommitResult(); + } uint32_t getHeartBeats() const { return _heartBeats; diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index 508a60480d0..421ebffafa4 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -8,6 +8,7 @@ using namespace proton::matching; using search::fef::FieldInfo; using search::fef::FieldType; using search::fef::Properties; +using search::fef::OnnxModel; using search::index::Schema; using search::index::schema::CollectionType; using search::index::schema::DataType; @@ -16,8 +17,8 @@ using SIAF = Schema::ImportedAttributeField; OnnxModels make_models() { OnnxModels::Vector list; - list.emplace_back("model1", "path1"); - list.emplace_back("model2", "path2"); + list.emplace_back(OnnxModel("model1", "path1").input_feature("input1","feature1").output_name("output1", "out1")); + list.emplace_back(OnnxModel("model2", "path2")); return OnnxModels(list); } @@ -104,10 +105,22 @@ TEST_F("require that imported attribute fields are extracted in index environmen EXPECT_EQUAL("[documentmetastore]", f.env.getField(2)->name()); } -TEST_F("require that onnx model paths can be obtained", Fixture(buildEmptySchema())) { - EXPECT_EQUAL(f1.env.getOnnxModelFullPath("model1").value(), vespalib::string("path1")); - EXPECT_EQUAL(f1.env.getOnnxModelFullPath("model2").value(), vespalib::string("path2")); - EXPECT_FALSE(f1.env.getOnnxModelFullPath("model3").has_value()); +TEST_F("require that onnx model config can be obtained", Fixture(buildEmptySchema())) { + { + auto model = f1.env.getOnnxModel("model1"); + ASSERT_TRUE(model != nullptr); + EXPECT_EQUAL(model->file_path(), vespalib::string("path1")); + EXPECT_EQUAL(model->input_feature("input1").value(), vespalib::string("feature1")); + EXPECT_EQUAL(model->output_name("output1").value(), vespalib::string("out1")); + } + { + auto model = f1.env.getOnnxModel("model2"); + ASSERT_TRUE(model != nullptr); + EXPECT_EQUAL(model->file_path(), vespalib::string("path2")); + EXPECT_FALSE(model->input_feature("input1").has_value()); + EXPECT_FALSE(model->output_name("output1").has_value()); + } + EXPECT_TRUE(f1.env.getOnnxModel("model3") == nullptr); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 1cc8d0280f6..c46990732b7 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/fef/indexproperties.h> +#include <vespa/searchlib/fef/onnx_model.h> #include <string> #include <vector> #include <map> @@ -18,6 +19,7 @@ const char *invalid_feature = "invalid_feature_name and format"; using namespace search::fef::indexproperties; using namespace search::index; +using search::fef::OnnxModel; using search::index::schema::CollectionType; using search::index::schema::DataType; @@ -69,9 +71,12 @@ struct Setup { std::map<std::string,std::string> properties; std::map<std::string,std::string> constants; std::vector<bool> extra_profiles; - std::map<std::string,std::string> onnx_models; + std::map<std::string,OnnxModel> onnx_models; Setup(); ~Setup(); + void add_onnx_model(const OnnxModel &model) { + onnx_models.insert_or_assign(model.name(), model); + } void index(const std::string &name, schema::DataType data_type, schema::CollectionType collection_type) { @@ -155,8 +160,20 @@ struct Setup { void write_onnx_models(const Writer &out) { size_t idx = 0; for (const auto &entry: onnx_models) { - out.fmt("model[%zu].name \"%s\"\n", idx, entry.first.c_str()); + out.fmt("model[%zu].name \"%s\"\n", idx, entry.second.name().c_str()); out.fmt("model[%zu].fileref \"onnx_ref_%zu\"\n", idx, idx); + size_t idx2 = 0; + for (const auto &input: entry.second.inspect_input_features()) { + out.fmt("model[%zu].input[%zu].name \"%s\"\n", idx, idx2, input.first.c_str()); + out.fmt("model[%zu].input[%zu].source \"%s\"\n", idx, idx2, input.second.c_str()); + ++idx2; + } + idx2 = 0; + for (const auto &output: entry.second.inspect_output_names()) { + out.fmt("model[%zu].output[%zu].name \"%s\"\n", idx, idx2, output.first.c_str()); + out.fmt("model[%zu].output[%zu].as \"%s\"\n", idx, idx2, output.second.c_str()); + ++idx2; + } ++idx; } } @@ -164,7 +181,7 @@ struct Setup { size_t idx = 0; for (const auto &entry: onnx_models) { out.fmt("file[%zu].ref \"onnx_ref_%zu\"\n", idx, idx); - out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.c_str()); + out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.file_path().c_str()); ++idx; } } @@ -225,7 +242,12 @@ struct SimpleSetup : Setup { struct OnnxSetup : Setup { OnnxSetup() : Setup() { - onnx_models["simple"] = TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx"); + add_onnx_model(OnnxModel("simple", TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx"))); + add_onnx_model(OnnxModel("mapped", TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx")) + .input_feature("query_tensor", "rankingExpression(qt)") + .input_feature("attribute_tensor", "rankingExpression(at)") + .input_feature("bias_tensor", "rankingExpression(bt)") + .output_name("output", "result")); } }; @@ -350,6 +372,13 @@ TEST_F("require that input type mismatch makes onnx model fail verification", On f.verify_invalid({"onnxModel(simple)"}); } +TEST_F("require that onnx model can have inputs and outputs mapped", OnnxSetup()) { + f.rank_expr("qt", "tensor<float>(a[1],b[4]):[[1,2,3,4]]"); + f.rank_expr("at", "tensor<float>(a[4],b[1]):[[5],[6],[7],[8]]"); + f.rank_expr("bt", "tensor<float>(a[1],b[1]):[[9]]"); + f.verify_valid({"onnxModel(mapped).result"}); +} + //----------------------------------------------------------------------------- TEST_F("cleanup files", Setup()) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index a49b27caf36..13695b969bf 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -212,7 +212,9 @@ void applyCommit(SerialNum serialNum, AttributeWriter::OnWriteDoneType , AttributeVector &attr) { if (attr.getStatus().getLastSyncToken() <= serialNum) { - attr.commit(serialNum, serialNum); + if (serialNum > attr.getCreateSerialNum()) { + attr.commit(serialNum, serialNum); + } } } diff --git a/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp index 1aa89b78966..73f909c115c 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/feeddebugger.cpp @@ -41,7 +41,7 @@ FeedDebugger::FeedDebugger() : _enableDebugging = ! (_debugLidList.empty() && _debugDocIdList.empty()); } -FeedDebugger::~FeedDebugger() {} +FeedDebugger::~FeedDebugger() = default; ns_log::Logger::LogLevel FeedDebugger::getDebugDebuggerInternal(uint32_t lid, const document::DocumentId * docid) const diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp index b04bac5ef26..1f979d1566c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.cpp @@ -7,6 +7,7 @@ namespace proton::documentmetastore { LidReuseDelayerConfig::LidReuseDelayerConfig(const DocumentDBConfig & configSnapshot) : _visibilityDelay(configSnapshot.getMaintenanceConfigSP()->getVisibilityDelay()), + _allowEarlyAck(configSnapshot.getMaintenanceConfigSP()->allowEarlyAck()), _hasIndexedOrAttributeFields(configSnapshot.getSchemaSP()->getNumIndexFields() > 0 || configSnapshot.getSchemaSP()->getNumAttributeFields() > 0) { @@ -18,6 +19,7 @@ LidReuseDelayerConfig::LidReuseDelayerConfig() LidReuseDelayerConfig::LidReuseDelayerConfig(vespalib::duration visibilityDelay, bool hasIndexedOrAttributeFields_in) : _visibilityDelay(visibilityDelay), + _allowEarlyAck(visibilityDelay > 1ms), _hasIndexedOrAttributeFields(hasIndexedOrAttributeFields_in) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h index 82dab433a22..c81a2ff399f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h @@ -15,6 +15,7 @@ class LidReuseDelayerConfig { private: vespalib::duration _visibilityDelay; + bool _allowEarlyAck; bool _hasIndexedOrAttributeFields; public: LidReuseDelayerConfig(); @@ -22,6 +23,7 @@ public: explicit LidReuseDelayerConfig(const DocumentDBConfig &configSnapshot); vespalib::duration visibilityDelay() const { return _visibilityDelay; } bool hasIndexedOrAttributeFields() const { return _hasIndexedOrAttributeFields; } + bool allowEarlyAck() const { return _allowEarlyAck; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp index 03dfd83a132..ed2202c830b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.cpp @@ -16,6 +16,7 @@ LidReuseDelayer::LidReuseDelayer(IThreadingService &writeService, IStore &docume : _writeService(writeService), _documentMetaStore(documentMetaStore), _immediateCommit(config.visibilityDelay() == vespalib::duration::zero()), + _allowEarlyAck(config.allowEarlyAck()), _config(config), _pendingLids() { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h index 5f1de878b4a..ba407ab57f8 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidreusedelayer.h @@ -27,6 +27,7 @@ class LidReuseDelayer searchcorespi::index::IThreadingService &_writeService; IStore &_documentMetaStore; const bool _immediateCommit; + const bool _allowEarlyAck; LidReuseDelayerConfig _config; std::vector<uint32_t> _pendingLids; // lids waiting for commit @@ -38,7 +39,8 @@ public: bool delayReuse(const std::vector<uint32_t> &lids); std::vector<uint32_t> getReuseLids(); - bool getImmediateCommit() const { return _immediateCommit; } + bool needImmediateCommit() const { return _immediateCommit; } + bool allowEarlyAck() const { return _allowEarlyAck; } const LidReuseDelayerConfig & getConfig() const { return _config; } }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp index 5743a3d44d6..013f359c4f9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp @@ -131,13 +131,10 @@ IndexEnvironment::hintFieldAccess(uint32_t ) const { } void IndexEnvironment::hintAttributeAccess(const string &) const { } -std::optional<vespalib::string> -IndexEnvironment::getOnnxModelFullPath(const vespalib::string &name) const +const search::fef::OnnxModel * +IndexEnvironment::getOnnxModel(const vespalib::string &name) const { - if (const auto model = _onnxModels.getModel(name)) { - return model->filePath; - } - return std::nullopt; + return _onnxModels.getModel(name); } IndexEnvironment::~IndexEnvironment() = default; diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h index d0e9a516cd0..ad51eb17b4d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h @@ -69,7 +69,7 @@ public: return _constantValueRepo.getConstant(name); } - std::optional<vespalib::string> getOnnxModelFullPath(const vespalib::string &name) const override; + const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const override; ~IndexEnvironment() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp index bdcf3e21d8e..ed80ca28bd6 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp @@ -1,25 +1,10 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "onnx_models.h" +#include <assert.h> namespace proton::matching { -OnnxModels::Model::Model(const vespalib::string &name_in, - const vespalib::string &filePath_in) - : name(name_in), - filePath(filePath_in) -{ -} - -OnnxModels::Model::~Model() = default; - -bool -OnnxModels::Model::operator==(const Model &rhs) const -{ - return (name == rhs.name) && - (filePath == rhs.filePath); -} - OnnxModels::OnnxModels() : _models() { @@ -30,15 +15,15 @@ OnnxModels::~OnnxModels() = default; OnnxModels::OnnxModels(const Vector &models) : _models() { - for (const auto &model : models) { - _models.insert(std::make_pair(model.name, model)); + for (const auto &model: models) { + _models.emplace(model.name(), model); } } bool OnnxModels::operator==(const OnnxModels &rhs) const { - return _models == rhs._models; + return (_models == rhs._models); } const OnnxModels::Model * @@ -51,4 +36,16 @@ OnnxModels::getModel(const vespalib::string &name) const return nullptr; } +void +OnnxModels::configure(const ModelConfig &config, Model &model) +{ + assert(config.name == model.name()); + for (const auto &input: config.input) { + model.input_feature(input.name, input.source); + } + for (const auto &output: config.output) { + model.output_name(output.name, output.as); + } +} + } diff --git a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h index fdaae657711..65ba524d8fc 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h +++ b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h @@ -3,6 +3,8 @@ #pragma once #include <vespa/vespalib/stllike/string.h> +#include <vespa/searchlib/fef/onnx_model.h> +#include <vespa/searchcore/config/config-onnx-models.h> #include <map> #include <vector> @@ -14,16 +16,8 @@ namespace proton::matching { */ class OnnxModels { public: - struct Model { - vespalib::string name; - vespalib::string filePath; - - Model(const vespalib::string &name_in, - const vespalib::string &filePath_in); - ~Model(); - bool operator==(const Model &rhs) const; - }; - + using ModelConfig = vespa::config::search::core::OnnxModelsConfig::Model; + using Model = search::fef::OnnxModel; using Vector = std::vector<Model>; private: @@ -38,6 +32,7 @@ public: bool operator==(const OnnxModels &rhs) const; const Model *getModel(const vespalib::string &name) const; size_t size() const { return _models.size(); } + static void configure(const ModelConfig &config, Model &model); }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index 066e135741e..ea5d46f02ad 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -218,6 +218,16 @@ CombiningFeedView::heartBeat(search::SerialNum serialNum) } } +bool +CombiningFeedView::allowEarlyAck() const { + for (const auto &view : _views) { + if ( ! view->allowEarlyAck() ) { + return false; + } + } + return true; +} + void CombiningFeedView::sync() { diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index d1da0408318..3a37fdc37cb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -83,6 +83,7 @@ public: // Called by document db executor void setCalculator(const IBucketStateCalculator::SP &newCalc); + bool allowEarlyAck() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 4c0485baec6..5ed0ad7492c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -135,6 +135,7 @@ public: } vespalib::duration getVisibilityDelay() const { return _visibilityDelay; } bool hasVisibilityDelay() const { return _visibilityDelay > vespalib::duration::zero(); } + bool allowEarlyAck() const { return _visibilityDelay > 1ms; } const DocumentDBLidSpaceCompactionConfig &getLidSpaceCompactionConfig() const { return _lidSpaceCompaction; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 46bcb0e49bb..c63785faa35 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -911,7 +911,16 @@ DocumentDB::syncFeedView() IFeedView::SP newFeedView(_subDBs.getFeedView()); _writeService.sync(); - _visibility.commit(); + /* + * Don't call commit() on visibility handler during transaction + * log replay since the serial number used for the commit will be + * too high until the replay is complete. This check can be + * removed again when feed handler has improved tracking of serial + * numbers during replay. + */ + if (_state.getAllowReconfig()) { + _visibility.commit(); + } _writeService.sync(); _feedView.set(newFeedView); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index a8996abc856..c8b701e82f8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -321,6 +321,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) LOG(info, "Got file path from file acquirer: '%s' (name='%s', ref='%s')", filePath.c_str(), rc.name.c_str(), rc.fileref.c_str()); models.emplace_back(rc.name, filePath); + OnnxModels::configure(rc, models.back()); } } newOnnxModels = std::make_shared<OnnxModels>(models); diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 734ef01d33a..8b82478c1a4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -58,17 +58,20 @@ class TlsMgrWriter : public TlsWriter { std::shared_ptr<search::transactionlog::Writer> _writer; public: TlsMgrWriter(TransactionLogManager &tls_mgr, - const search::transactionlog::WriterFactory & factory) : - _tls_mgr(tls_mgr), - _writer(factory.getWriter(tls_mgr.getDomainName())) + const search::transactionlog::WriterFactory & factory) + : _tls_mgr(tls_mgr), + _writer(factory.getWriter(tls_mgr.getDomainName())) { } void appendOperation(const FeedOperation &op, DoneCallback onDone) override; + [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override { + return _writer->startCommit(std::move(onDone)); + } bool erase(SerialNum oldest_to_keep) override; SerialNum sync(SerialNum syncTo) override; }; - -void TlsMgrWriter::appendOperation(const FeedOperation &op, DoneCallback onDone) { +void +TlsMgrWriter::appendOperation(const FeedOperation &op, DoneCallback onDone) { using Packet = search::transactionlog::Packet; vespalib::nbostream stream; op.serialize(stream); @@ -77,9 +80,11 @@ void TlsMgrWriter::appendOperation(const FeedOperation &op, DoneCallback onDone) Packet::Entry entry(op.getSerialNum(), op.getType(), vespalib::ConstBufferRef(stream.data(), stream.size())); Packet packet(entry.serializedSize()); packet.add(entry); - _writer->commit(packet, std::move(onDone)); + _writer->append(packet, std::move(onDone)); } -bool TlsMgrWriter::erase(SerialNum oldest_to_keep) { + +bool +TlsMgrWriter::erase(SerialNum oldest_to_keep) { return _tls_mgr.getSession()->erase(oldest_to_keep); } @@ -88,22 +93,40 @@ TlsMgrWriter::sync(SerialNum syncTo) { for (int retryCount = 0; retryCount < 10; ++retryCount) { SerialNum syncedTo(0); - LOG(spam, "Trying tls sync(%" PRIu64 ")", syncTo); + LOG(debug, "Trying tls sync(%" PRIu64 ")", syncTo); bool res = _tls_mgr.getSession()->sync(syncTo, syncedTo); if (!res) { - LOG(spam, "Tls sync failed, retrying"); + LOG(debug, "Tls sync failed, retrying"); sleep(1); continue; } if (syncedTo >= syncTo) { - LOG(spam, "Tls sync complete, reached %" PRIu64", returning", syncedTo); + LOG(debug, "Tls sync complete, reached %" PRIu64", returning", syncedTo); return syncedTo; } - LOG(spam, "Tls sync incomplete, reached %" PRIu64 ", retrying", syncedTo); + LOG(debug, "Tls sync incomplete, reached %" PRIu64 ", retrying", syncedTo); } throw IllegalStateException(make_string("Failed to sync TLS to token %" PRIu64 ".", syncTo)); } +class OnCommitDone : public search::IDestructorCallback { +public: + OnCommitDone(Executor & executor, std::unique_ptr<Executor::Task> task) + : _executor(executor), + _task(std::move(task)) + {} + ~OnCommitDone() override { _executor.execute(std::move(_task)); } +private: + Executor & _executor; + std::unique_ptr<Executor::Task> _task; +}; + +template <typename T> +struct KeepAlive : public search::IDestructorCallback { + explicit KeepAlive(T toKeep) : _toKeep(std::move(toKeep)) { } + ~KeepAlive() override = default; + T _toKeep; +}; } // namespace void @@ -379,6 +402,9 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), + _numOperationsPendingCommit(0), + _numOperationsCompleted(0), + _numCommitsCompleted(0), _delayedPrune(false), _feedLock(), _feedState(make_shared<InitState>(getDocTypeName())), @@ -472,17 +498,57 @@ FeedHandler::getTransactionLogReplayDone() const { } void +FeedHandler::onCommitDone(size_t numPendingAtStart) { + assert(numPendingAtStart <= _numOperationsPendingCommit); + _numOperationsPendingCommit -= numPendingAtStart; + _numOperationsCompleted += numPendingAtStart; + _numCommitsCompleted++; + if (_numOperationsPendingCommit > 0) { + enqueCommitTask(); + } + LOG(spam, "%zu: onCommitDone(%zu) total=%zu left=%zu", + _numCommitsCompleted, numPendingAtStart, _numOperationsCompleted, _numOperationsPendingCommit); +} + +void FeedHandler::enqueCommitTask() { + _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); +} + +void +FeedHandler::initiateCommit() { + auto onCommitDoneContext = std::make_shared<OnCommitDone>( + _writeService.master(), + makeLambdaTask([this, numPendingAtStart=_numOperationsPendingCommit]() { + onCommitDone(numPendingAtStart); + })); + auto commitResult = _tlsWriter->startCommit(onCommitDoneContext); + if (_activeFeedView && ! _activeFeedView->allowEarlyAck()) { + using KeepAlivePair = KeepAlive<std::pair<CommitResult, DoneCallback>>; + auto pair = std::make_pair(std::move(commitResult), std::move(onCommitDoneContext)); + _activeFeedView->forceCommit(_serialNum, std::make_shared<KeepAlivePair>(std::move(pair))); + } +} + +void FeedHandler::appendOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { const_cast<FeedOperation &>(op).setSerialNum(incSerialNum()); } _tlsWriter->appendOperation(op, std::move(onDone)); + if (++_numOperationsPendingCommit == 1) { + enqueCommitTask(); + } +} + +FeedHandler::CommitResult +FeedHandler::startCommit(DoneCallback onDone) { + return _tlsWriter->startCommit(std::move(onDone)); } void FeedHandler::storeOperationSync(const FeedOperation &op) { vespalib::Gate gate; - appendOperation(op, make_shared<search::GateCallback>(gate)); + appendAndCommitOperation(op, make_shared<search::GateCallback>(gate)); gate.await(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 29961f4a6cc..c295b26a759 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -76,6 +76,9 @@ private: // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; + size_t _numOperationsPendingCommit; + size_t _numOperationsCompleted; + size_t _numCommitsCompleted; bool _delayedPrune; mutable std::shared_mutex _feedLock; FeedStateSP _feedState; @@ -125,6 +128,9 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); + void onCommitDone(size_t numPendingAtStart); + void initiateCommit(); + void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; @@ -226,6 +232,7 @@ public: void performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp) override; void syncTls(SerialNum syncTo); void appendOperation(const FeedOperation &op, DoneCallback onDone) override; + [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override; void storeOperationSync(const FeedOperation & op); void considerDelayedPrune(); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.h b/searchcore/src/vespa/searchcore/proton/server/feedstate.h index 6de1d7a4322..7d559eb4375 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.h @@ -11,7 +11,7 @@ namespace vespalib { namespace proton { class FeedOperation; -class PacketWrapper; +struct PacketWrapper; /** * Class representing the current state of a feed handler. diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index 47b81a9a17f..c3b76a9db75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -14,12 +14,18 @@ class FeedOperation; struct IOperationStorer { using DoneCallback = search::transactionlog::Writer::DoneCallback; + using CommitResult = search::transactionlog::Writer::CommitResult; virtual ~IOperationStorer() = default; /** * Assign serial number to (if not set) and store the given operation. */ virtual void appendOperation(const FeedOperation &op, DoneCallback onDone) = 0; + [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; + void appendAndCommitOperation(const FeedOperation &op, DoneCallback onDone) { + appendOperation(op, onDone); + (void) startCommit(std::move(onDone)); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h index 4b028a289a9..8dd7ae8474e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h @@ -65,6 +65,7 @@ public: virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation & pruneOp) = 0; virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0; virtual ILidCommitState & getUncommittedLidsTracker() = 0; + virtual bool allowEarlyAck() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index d423e095ad9..468850b4409 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -82,7 +82,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); vespalib::Gate gate; - _opStorer.appendOperation(op, std::make_shared<search::GateCallback>(gate)); + _opStorer.appendAndCommitOperation(op, std::make_shared<search::GateCallback>(gate)); gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h index 37497eaa998..35549f21471 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.h @@ -63,13 +63,13 @@ public: ~LidSpaceCompactionJob(); // Implements IDiskMemUsageListener - virtual void notifyDiskMemUsage(DiskMemUsageState state) override; + void notifyDiskMemUsage(DiskMemUsageState state) override; // Implements IClusterStateChangedNofifier - virtual void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; + void notifyClusterStateChanged(const IBucketStateCalculator::SP &newCalc) override; // Implements IMaintenanceJob - virtual bool run() override; + bool run() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index d94bb2e3d03..d4b542a0af8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -105,7 +105,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique<PruneSessionCacheJob>(scPruner, config.getSessionCachePruneInterval())); - if (config.hasVisibilityDelay()) { + if (config.hasVisibilityDelay() && config.allowEarlyAck()) { controller.registerJobInMasterThread(std::make_unique<DocumentDBCommitJob>(commit, config.getVisibilityDelay())); } const MaintenanceDocumentSubDB &mRemSubDB(controller.getRemSubDB()); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 217a3bb24d3..90875aa8591 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -210,11 +210,11 @@ moveMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id, c } std::unique_ptr<PendingLidTrackerBase> -createUncommitedLidTracker(bool needImmediateCommit) { - if (needImmediateCommit) { - return std::make_unique<PendingLidTracker>(); - } else { +createUncommitedLidTracker(bool allowEarlyAck) { + if (allowEarlyAck) { return std::make_unique<TwoPhasePendingLidTracker>(); + } else { + return std::make_unique<PendingLidTracker>(); } } @@ -229,7 +229,7 @@ StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx, const PersistentParams _docType(nullptr), _lidReuseDelayer(ctx._writeService, _documentMetaStoreContext->get(), ctx._lidReuseDelayerConfig), _pendingLidsForDocStore(), - _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.getImmediateCommit())), + _pendingLidsForCommit(createUncommitedLidTracker(_lidReuseDelayer.allowEarlyAck())), _schema(ctx._schema), _writeService(ctx._writeService), _params(params), @@ -263,7 +263,7 @@ StoreOnlyFeedView::forceCommit(SerialNum serialNum, DoneCallback onDone) void StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) { - (void) serialNum; + LOG(debug, "internalForceCommit: serial=%" PRIu64 ".", serialNum); _writeService.summary().execute(makeLambdaTask([onDone=onCommitDone]() {(void) onDone;})); std::vector<uint32_t> lidsToReuse; lidsToReuse = _lidReuseDelayer.getReuseLids(); @@ -275,7 +275,7 @@ StoreOnlyFeedView::internalForceCommit(SerialNum serialNum, OnForceCommitDoneTyp void StoreOnlyFeedView::considerEarlyAck(FeedToken & token) { - if ( ! needCommit() && token) { + if (allowEarlyAck() && token) { token.reset(); } } @@ -327,7 +327,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), @@ -345,8 +345,13 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) } bool -StoreOnlyFeedView::needCommit() const { - return _lidReuseDelayer.getImmediateCommit(); +StoreOnlyFeedView::needImmediateCommit() const { + return _lidReuseDelayer.needImmediateCommit(); +} + +bool +StoreOnlyFeedView::allowEarlyAck() const { + return _lidReuseDelayer.allowEarlyAck(); } void @@ -483,7 +488,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) auto uncommitted = _pendingLidsForCommit->produce(updOp.getLid()); considerEarlyAck(token); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), std::move(uncommitted), updOp.getUpdate()); UpdateScope updateScope(*_schema, upd); updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); @@ -657,7 +662,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token unc std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone); } @@ -770,7 +775,7 @@ StoreOnlyFeedView::handleDeleteBucket(const DeleteBucketOperation &delOp) void StoreOnlyFeedView::internalDeleteBucket(const DeleteBucketOperation &delOp) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); size_t rm_count = removeDocuments(delOp, true, immediateCommit); LOG(debug, "internalDeleteBucket(): docType(%s), bucket(%s), lidsToRemove(%zu)", _params._docTypeName.toString().c_str(), delOp.getBucketId().toString().c_str(), rm_count); @@ -809,7 +814,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - bool immediateCommit = needCommit(); + bool immediateCommit = needImmediateCommit(); const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 7d91ea86a22..20942423995 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -161,7 +161,7 @@ private: void putSummary(SerialNum serialNum, Lid lid, DocumentSP doc, OnOperationDoneType onDone); void removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone); void heartBeatSummary(SerialNum serialNum); - bool needCommit() const; + bool needImmediateCommit() const; bool useDocumentStore(SerialNum replaySerialNum) const { @@ -264,6 +264,7 @@ public: void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &pruneOp) override; void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; ILidCommitState & getUncommittedLidsTracker() override; + bool allowEarlyAck() const final override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h index adfc911c8df..b96fd77409c 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h @@ -34,6 +34,7 @@ struct DummyFeedView : public IFeedView void handleCompactLidSpace(const CompactLidSpaceOperation &) override {} void forceCommit(search::SerialNum, DoneCallback) override { } ILidCommitState & getUncommittedLidsTracker() override; + bool allowEarlyAck() const override { return false; } }; } diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 1f8c054c694..33b78d517a3 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -102,7 +102,6 @@ vespa_define_module( src/tests/bitcompression/expgolomb src/tests/bitvector src/tests/btree - src/tests/bytecomplens src/tests/common/bitvector src/tests/common/location src/tests/common/location_iterator diff --git a/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp b/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp index 1e89fbe7501..aca9b652b94 100644 --- a/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp +++ b/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp @@ -72,6 +72,8 @@ TEST(AttributeHeaderTest, can_be_added_to_and_extracted_from_generic_header) verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Euclidean})); verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Angular})); verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::GeoDegrees})); + verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::InnerProduct})); + verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Hamming})); verify_roundtrip_serialization(HnswIPO()); } diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index eeabbf23c27..1e3531dc78e 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/attribute/attributefilewriter.h> #include <vespa/searchlib/attribute/attributefilebufferwriter.h> #include <vespa/searchlib/attribute/attribute_header.h> @@ -10,6 +9,7 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index d8761f69d71..2a5b8014299 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -10,7 +10,6 @@ #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributesaver.h> #include <vespa/searchlib/attribute/multinumericattribute.h> -#include <vespa/searchlib/attribute/multistringattribute.h> #include <vespa/searchlib/attribute/singlenumericattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/queryeval/executeinfo.h> @@ -21,6 +20,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/compress.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/searchlib/attribute/attributevector.hpp> diff --git a/searchlib/src/tests/bytecomplens/.gitignore b/searchlib/src/tests/bytecomplens/.gitignore deleted file mode 100644 index afe9bff02f6..00000000000 --- a/searchlib/src/tests/bytecomplens/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.So -.depend* -Makefile -bytecomp_test -searchlib_bytecomp_test_app diff --git a/searchlib/src/tests/bytecomplens/CMakeLists.txt b/searchlib/src/tests/bytecomplens/CMakeLists.txt deleted file mode 100644 index 24ecef59b15..00000000000 --- a/searchlib/src/tests/bytecomplens/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_bytecomp_test_app TEST - SOURCES - bytecomp.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_bytecomp_test_app NO_VALGRIND COMMAND searchlib_bytecomp_test_app) diff --git a/searchlib/src/tests/bytecomplens/bytecomp.cpp b/searchlib/src/tests/bytecomplens/bytecomp.cpp deleted file mode 100644 index adc92b0097d..00000000000 --- a/searchlib/src/tests/bytecomplens/bytecomp.cpp +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <memory> -#include <vespa/log/log.h> -LOG_SETUP("bytecomplens_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/random.h> -#include <vespa/searchlib/docstore/bytecomplens.h> - - -class Test : public vespalib::TestApp { -private: - void testRandomLengths(); - -public: - int Main() override { - TEST_INIT("bytecomplens_test"); - testRandomLengths(); TEST_FLUSH(); - TEST_DONE(); - } -}; - -TEST_APPHOOK(Test); - - -void -Test::testRandomLengths() -{ - vespalib::RandomGen rndgen(0x07031969); - -#define TBLSIZ 0xc00000 - - auto lentable = std::unique_ptr<uint32_t[]>(new uint32_t[TBLSIZ]); - auto offtable = std::unique_ptr<uint64_t[]>(new uint64_t[TBLSIZ]); - - uint64_t offset = 16; - - for (int i = 0; i < TBLSIZ; i++) { - int sel = rndgen.nextInt32(); - int val = rndgen.nextInt32(); - switch (sel & 0x7) { - case 0: - val &= 0x7F; - break; - case 1: - val &= 0xFF; - break; - case 3: - val &= 0x1FFF; - break; - case 4: - val &= 0x3FFF; - break; - case 5: - val &= 0x7FFF; - break; - case 6: - val &= 0xFFFF; - break; - case 7: - default: - val &= 0xFFFFF; - break; - } - offtable[i] = offset; - lentable[i] = val; - offset += val; - } - - LOG(info, "made %d random offsets", TBLSIZ); - - search::ByteCompressedLengths foo; - - LOG(info, "empty BCL using %9ld bytes memory", foo.memoryUsed()); - - foo.addOffsetTable(TBLSIZ/4, offtable.get()); - foo.addOffsetTable(TBLSIZ/4, offtable.get() + 1*(TBLSIZ/4)); - - LOG(info, "half BCL using %9ld bytes memory", foo.memoryUsed()); - - search::ByteCompressedLengths bar; - foo.swap(bar); - bar.addOffsetTable(TBLSIZ/4, offtable.get() + 2*(TBLSIZ/4)); - bar.addOffsetTable(TBLSIZ/4, offtable.get() + 3*(TBLSIZ/4)); - foo.swap(bar); - - LOG(info, "full BCL using %9ld bytes memory", foo.memoryUsed()); - - LOG(info, "constructed %d byte compressed lengths", TBLSIZ-1); - - for (int i = 0; i < TBLSIZ-1; i++) { - search::ByteCompressedLengths::OffLen offlen; - offlen = foo.getOffLen(i); - - if ((i % 1000000) == 0) { - LOG(info, "data blob [%d] length %" PRIu64 " offset %" PRIu64, i, offlen.length, offlen.offset); - } - EXPECT_EQUAL(lentable[i], offlen.length); - EXPECT_EQUAL(offtable[i], offlen.offset); - } -} - diff --git a/searchlib/src/tests/bytecomplens/example.txt b/searchlib/src/tests/bytecomplens/example.txt deleted file mode 100644 index 6dc3df0118a..00000000000 --- a/searchlib/src/tests/bytecomplens/example.txt +++ /dev/null @@ -1,122 +0,0 @@ -offset length BCN val L0 len/off skipL1 skipL2 skipL3 - -976 18707 [ 93 92 01 ] 3/0 976/0/0/0 -19683 11527 [ 87 5A ] 2/3 -31210 3926 [ D6 1E ] 2/5 -35136 2 [ 02 ] 1/7 -35138 6060 [ AC 2F ] 2/8 34162/8 -41198 649445 [ E5 D1 27 ] 3/10 -690643 2866 [ B2 16 ] 2/13 -693509 824767 [ BF AB 32 ] 3/15 -1518276 499173 [ E5 BB 1E ] 3/18 1483138/10 -2017449 20455 [ E7 9F 01 ] 3/21 -2037904 11 [ 0B ] 1/24 -2037915 19207 [ 87 96 01 ] 3/25 -2057122 6355 [ D3 31 ] 2/28 538846/10 -2063477 3422 [ DE 1A ] 2/30 -2066899 10683 [ BB 53 ] 2/32 -2077582 7360 [ C0 39 ] 2/34 -2084942 17969 [ B1 8C 01 ] 3/36 2083966/36/12 -2102911 6114 [ E2 2F ] 2/39 -2109025 31741 [ FD F7 01 ] 3/41 -2140766 581588 [ D4 BF 23 ] 3/44 -2722354 5341 [ DD 29 ] 2/47 637412/11 -2727695 13774 [ CE 6B ] 2/49 -2741469 717809 [ F1 E7 2B ] 3/51 -3459278 815406 [ AE E2 31 ] 3/54 -4274684 89 [ 59 ] 1/57 1552330/10 -4274773 4545 [ C1 23 ] 2/58 -4279318 803868 [ 9C 88 31 ] 3/60 -5083186 12865 [ C1 64 ] 2/63 -5096051 75 [ 4B ] 1/65 821367/8 -5096126 40734 [ 9E BE 02 ] 3/66 -5136860 101 [ 65 ] 1/69 -5136961 128 [ 80 01 ] 2/70 -5137089 253 [ FD 01 ] 2/72 3052147/36/12 -5137342 13 [ 0D ] 1/74 -5137355 24986 [ 9A C3 01 ] 3/75 -5162341 231 [ E7 01 ] 2/78 -5162572 997853 [ DD F3 3C ] 3/80 25483/8 -6160425 4728 [ F8 24 ] 2/83 -6165153 2025 [ E9 0F ] 2/85 -6167178 7281 [ F1 38 ] 2/87 -6174459 1026302 [ FE D1 3E ] 3/89 1011887/9 -7200761 848783 [ 8F E7 33 ] 3/92 -8049544 145767 [ E7 F2 08 ] 3/95 -8195311 19103 [ 9F 95 01 ] 3/98 -8214414 22166 [ 96 AD 01 ] 3/101 2039955/12 -8236580 30020 [ C4 EA 01 ] 3/104 -8266600 13 [ 0D ] 1/107 -8266613 120 [ 78 ] 1/108 -8266733 22398 [ FE AE 01 ] 3/109 3129644/37/12 -8289131 10832 [ D0 54 ] 2/112 -8299963 3765 [ B5 1D ] 2/114 -8303728 432771 [ 83 B5 1A ] 3/116 -8736499 30133 [ B5 EB 01 ] 3/119 469766/10 -8766632 6444 [ AC 32 ] 2/122 -8773076 16033 [ A1 7D ] 2/124 -8789109 78 [ 4E ] 1/126 -8789187 12510 [ DE 61 ] 2/127 52688/8 -8801697 12441 [ 99 61 ] 2/129 -8814138 117 [ 75 ] 1/131 -8814255 7147 [ EB 37 ] 2/132 -8821402 189 [ BD 01 ] 2/134 32215/7 -8821591 199704 [ 98 98 0C ] 3/136 -9021295 13240 [ B8 67 ] 2/139 -9034535 110 [ 6E ] 1/141 -9034645 31677 [ BD F7 01 ] 3/142 9034645/142/48/17 -9066322 18547 [ F3 90 01 ] 3/145 -9084869 734679 [ D7 EB 2C ] 3/148 -9819548 112 [ 70 ] 1/151 -9819660 883565 [ ED F6 35 ] 3/152 785015/10 -10703225 10290 [ B2 50 ] 2/155 -10713515 21410 [ A2 A7 01 ] 3/157 -10734925 15 [ 0F ] 1/160 -10734940 747774 [ FE D1 2D ] 3/161 915280/9 -11482714 39 [ 27 ] 1/164 -11482753 77 [ 4D ] 1/165 -11482830 235 [ EB 01 ] 2/166 -11483065 1991 [ C7 0F ] 2/168 748125/7 -11485056 9187 [ E3 47 ] 2/170 -11494243 18800 [ F0 92 01 ] 3/172 -11513043 1042219 [ AB CE 3F ] 3/175 -12555262 9154 [ C2 47 ] 2/178 3520617/36/12 -12564416 43582 [ BE D4 02 ] 3/180 -12607998 847240 [ 88 DB 33 ] 3/183 -13455238 4726 [ F6 24 ] 2/186 -13459964 590348 [ 8C 84 24 ] 3/188 904702/10 -14050312 8659 [ D3 43 ] 2/191 -14058971 116 [ 74 ] 1/193 -14059087 13563 [ FB 69 ] 2/194 -14072650 713064 [ E8 C2 2B ] 3/196 612686/8 -14785714 40321 [ 81 BB 02 ] 3/199 -14826035 2296 [ F8 11 ] 2/202 -14828331 7273 [ E9 38 ] 2/204 -14835604 68285 [ BD 95 04 ] 3/206 762954/10 -14903889 235 [ EB 01 ] 2/209 -14904124 4669 [ BD 24 ] 2/211 -14908793 28535 [ F7 DE 01 ] 3/213 -14937328 19 [ 13 ] 1/216 2382066/38/12 -14937347 5369 [ F9 29 ] 2/217 -14942716 602191 [ CF E0 24 ] 3/219 -15544907 2653 [ DD 14 ] 2/222 -15547560 25755 [ 9B C9 01 ] 3/224 610232/8 -15573315 11349 [ D5 58 ] 2/227 -15584664 15006 [ 9E 75 ] 2/229 -15599670 89 [ 59 ] 1/231 -15599759 52772 [ A4 9C 03 ] 3/232 52199/8 -15652531 776175 [ EF AF 2F ] 3/235 -16428706 126 [ 7E ] 1/238 -16428832 3884 [ AC 1E ] 2/239 -16432716 33958 [ A6 89 02 ] 3/241 832957/9 -16466674 122 [ 7A ] 1/244 -16466796 41895 [ A7 C7 02 ] 3/245 -16508691 105882 [ 9A BB 06 ] 3/248 -16614573 11067 [ BB 56 ] 2/251 1677245/35/12 -16625640 4588 [ EC 23 ] 2/253 -16630228 7349 [ B5 39 ] 2/255 -16637577 902638 [ EE 8B 37 ] 3/257 -17540215 8737 [ A1 44 ] 2/260 925642/9 -17548952 29186 [ 82 E4 01 ] 3/262 -17578138 41 [ 29 ] 1/265 -17578179 diff --git a/searchlib/src/tests/bytecomplens/tblprint.cpp b/searchlib/src/tests/bytecomplens/tblprint.cpp deleted file mode 100644 index 6a7347f3c0d..00000000000 --- a/searchlib/src/tests/bytecomplens/tblprint.cpp +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("tblprint"); -#include <vespa/vespalib/util/random.h> - -#include <vector> -#include <vespa/vespalib/data/databuffer.h> - - -/** - * Class compressing a table of offsets in memory. - * After adding (n) offsets you can access - * (n-1) pairs of (length, offset). - * All offsets must be increasing, but they - * may be added in several chunks. - **/ -class ByteCompressedLengths -{ -public: - /** - * Construct an empty instance - **/ - ByteCompressedLengths(); - - /** - * add the given offset table. - * @param entries number of offsets to store. - * @param offsets table that contains (entries) offsets. - **/ - void addOffsetTable(uint64_t entries, uint64_t *offsets); - - /** - * free resources - **/ - ~ByteCompressedLengths(); - - /** - * Fetch a length and offset from compressed data. - * Note invariant: id < size(); size() == (entries-1) - * - * @param id The index into the offset table - * @param offset Will be incremented by offset[id] - * @return The delta (offset[id+1] - offset[id]) - **/ - uint64_t getLength(uint64_t id, uint64_t &offset) const; - - /** - * The number of (length, offset) pairs stored - **/ - uint64_t size() const { return _entries; } - - struct L3Entry { - uint64_t offset; - uint64_t l0toff; - uint64_t l1toff; - uint64_t l2toff; - }; - vespalib::DataBuffer _l0space; - vespalib::DataBuffer _l1space; - vespalib::DataBuffer _l2space; - const uint8_t *_l0table; - const uint8_t *_l1table; - const uint8_t *_l2table; - - std::vector<L3Entry> _l3table; - - uint64_t _lenSum1; - uint64_t _lenSum2; - uint64_t _l0oSum1; - uint64_t _l0oSum2; - uint64_t _l1oSum2; - uint64_t _last_offset; - uint64_t _entries; - - void addOffset(uint64_t offset); -}; - -/** - * get "Byte Compressed Number" from buffer, incrementing pointer - **/ -static inline uint64_t getBCN(const uint8_t *&buffer) -{ - uint8_t b = *buffer++; - uint64_t len = (b & 127); - unsigned shiftLen = 0; - while (b & 128) { - shiftLen += 7; - b = *buffer++; - len |= ((b & 127) << shiftLen); - } - return len; -} - -static size_t writeLen(vespalib::DataBuffer &buf, uint64_t len) -{ - size_t bytes = 0; - do { - uint8_t b = len & 127; - len >>= 7; - if (len > 0) { - b |= 128; - } - buf.ensureFree(1); - buf.writeInt8(b); - ++bytes; - } while (len > 0); - return bytes; -} - - -ByteCompressedLengths::ByteCompressedLengths() - : _l0space(), - _l1space(), - _l2space(), - _l3table(), - _lenSum1(0), - _lenSum2(0), - _l0oSum1(0), - _l0oSum2(0), - _l1oSum2(0), - _last_offset(0), - _entries(0) -{ -} - - -void -ByteCompressedLengths::addOffset(uint64_t offset) -{ - assert(offset >= _last_offset); - - uint64_t len = offset - _last_offset; - uint64_t i = _entries++; - - if ((i & 3) == 0) { - _lenSum2 += _lenSum1; - _l0oSum2 += _l0oSum1; - - uint64_t t1n = i >> 2; - if ((t1n & 3) == 0) { - uint64_t t2n = t1n >> 2; - - if ((t2n & 3) == 0) { - L3Entry e; - e.offset = _last_offset; - e.l0toff = _l0space.getDataLen(); - e.l1toff = _l1space.getDataLen(); - e.l2toff = _l2space.getDataLen(); - - _l3table.push_back(e); - } else { - writeLen(_l2space, _lenSum2); - writeLen(_l2space, _l0oSum2); - writeLen(_l2space, _l1oSum2); - } - _lenSum2 = 0; - _l0oSum2 = 0; - _l1oSum2 = 0; - } else { - _l1oSum2 += writeLen(_l1space, _lenSum1); - _l1oSum2 += writeLen(_l1space, _l0oSum1); - } - _lenSum1 = 0; - _l0oSum1 = 0; - } - _l0oSum1 += writeLen(_l0space, len); - _lenSum1 += len; - _last_offset = offset; -} - - -void -ByteCompressedLengths::addOffsetTable(uint64_t entries, uint64_t *offsets) -{ - if (entries == 0) return; - // Do we have some offsets already? - if (_entries > 0) { - // yes, add first offset normally - addOffset(offsets[0]); - } else { - // no, special treatment for very first offset - _last_offset = offsets[0]; - } - for (uint64_t cnt = 1; cnt < entries; ++cnt) { - addOffset(offsets[cnt]); - } - _l0table = (uint8_t *)_l0space.getData(); - _l1table = (uint8_t *)_l1space.getData(); - _l2table = (uint8_t *)_l2space.getData(); - - LOG(debug, "compressed %ld offsets", (_entries+1)); - LOG(debug, "(%ld bytes)", (_entries+1)*sizeof(uint64_t)); - LOG(debug, "to (%ld + %ld + %ld) bytes + %ld l3entries", - _l0space.getDataLen(), - _l1space.getDataLen(), - _l2space.getDataLen(), - _l3table.size()); - LOG(debug, "(%ld bytes)", - (_l0space.getDataLen() + _l1space.getDataLen() + _l2space.getDataLen() + - _l3table.size()*sizeof(L3Entry))); -} - - -ByteCompressedLengths::~ByteCompressedLengths() -{ -} - -uint64_t -ByteCompressedLengths::getLength(uint64_t numSkip, uint64_t &offset) const -{ - assert(numSkip < _entries); - - unsigned skipL0 = numSkip & 3; - unsigned skipL1 = (numSkip >> 2) & 3; - unsigned skipL2 = (numSkip >> 4) & 3; - uint64_t skipL3 = (numSkip >> 6); - - offset += _l3table[skipL3].offset; - uint64_t l0toff = _l3table[skipL3].l0toff; - uint64_t l1toff = _l3table[skipL3].l1toff; - uint64_t l2toff = _l3table[skipL3].l2toff; - - // printf("start off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - - const uint8_t *l2pos = _l2table + l2toff; - - while (skipL2 > 0) { - --skipL2; - offset += getBCN(l2pos); - l0toff += getBCN(l2pos); - l1toff += getBCN(l2pos); - } - - const uint8_t *l1pos = _l1table + l1toff; - - while (skipL1 > 0) { - --skipL1; - offset += getBCN(l1pos); - l0toff += getBCN(l1pos); - - } - const uint8_t *l0pos = _l0table + l0toff; - - while (skipL0 > 0) { - --skipL0; - offset += getBCN(l0pos); - } - // printf("end off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - return getBCN(l0pos); -} - - - -class Test { -public: - static void printTable(); -}; - - - -int main(int /*argc*/, char ** /*argv*/) -{ - Test::printTable(); - return 0; -} - -void -Test::printTable() -{ - vespalib::RandomGen rndgen(0x07031969); -#define TBLSIZ 120 - uint32_t *lentable = new uint32_t[TBLSIZ]; - uint64_t *offtable = new uint64_t[TBLSIZ]; - - uint64_t offset = 16 + TBLSIZ*8; - - for (int i = 0; i < TBLSIZ; i++) { - int sel = rndgen.nextInt32(); - int val = rndgen.nextInt32(); - switch (sel & 0x7) { - case 0: - val &= 0x7F; - break; - case 1: - val &= 0xFF; - break; - case 3: - val &= 0x1FFF; - break; - case 4: - val &= 0x3FFF; - break; - case 5: - val &= 0x7FFF; - break; - case 6: - val &= 0xFFFF; - break; - case 7: - default: - val &= 0xFFFFF; - break; - } - offtable[i] = offset; - lentable[i] = val; - offset += val; - } - - ByteCompressedLengths foo; - foo.addOffsetTable(TBLSIZ, offtable); - - const uint8_t *l1pos = foo._l1table; - const uint8_t *l2pos = foo._l2table; - - printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - "offset", "length", "BCN val", "L0 len/off", "skipL1", "skipL2", "skipL3"); - - int slb = 0; - for (int i = 0; i+1 < TBLSIZ; i++) { - printf("%ld\t%d\t[", offtable[i], lentable[i]); - int bytes=0; - uint64_t len = lentable[i]; - do { - uint8_t b = len & 127; - len >>= 7; - if (len > 0) { - b |= 128; - } - printf(" %02X", b); - ++bytes; - } while (len > 0); - printf(" ]\t%d", bytes); - printf("/%d", slb); - slb += bytes; - - if ((i & 63) == 0) { - printf("\t\t\t%ld/%ld/%ld/%ld", - foo._l3table[i >> 6].offset, - foo._l3table[i >> 6].l0toff, - foo._l3table[i >> 6].l1toff, - foo._l3table[i >> 6].l2toff); - } else - if ((i & 15) == 0) { - printf("\t\t%ld", getBCN(l2pos)); - printf("/%ld", getBCN(l2pos)); - printf("/%ld", getBCN(l2pos)); - } else - if ((i & 3) == 0) { - printf("\t%ld", getBCN(l1pos)); - printf("/%ld", getBCN(l1pos)); - } - printf("\n"); - } - printf("%ld\n", offtable[TBLSIZ-1]); - fflush(stdout); -} diff --git a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp index b49d9c365de..6a1e4ef9fa1 100644 --- a/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp +++ b/searchlib/src/tests/features/onnx_feature/onnx_feature_test.cpp @@ -58,8 +58,8 @@ struct OnnxFeatureTest : ::testing::Test { vespalib::string expr_name = feature_name + ".rankingScript"; indexEnv.getProperties().add(expr_name, expr); } - void add_onnx(const vespalib::string &name, const vespalib::string &file) { - indexEnv.addOnnxModel(name, file); + void add_onnx(const OnnxModel &model) { + indexEnv.addOnnxModel(model); } void compile(const vespalib::string &seed) { resolver->addSeed(seed); @@ -89,7 +89,7 @@ TEST_F(OnnxFeatureTest, simple_onnx_model_can_be_calculated) { add_expr("query_tensor", "tensor<float>(a[1],b[4]):[[docid,2,3,4]]"); add_expr("attribute_tensor", "tensor<float>(a[4],b[1]):[[5],[6],[7],[8]]"); add_expr("bias_tensor", "tensor<float>(a[1],b[1]):[[9]]"); - add_onnx("simple", simple_model); + add_onnx(OnnxModel("simple", simple_model)); compile(onnx_feature("simple")); EXPECT_EQ(get(1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); EXPECT_EQ(get("onnxModel(simple).output", 1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); @@ -101,7 +101,7 @@ TEST_F(OnnxFeatureTest, dynamic_onnx_model_can_be_calculated) { add_expr("query_tensor", "tensor<float>(a[1],b[4]):[[docid,2,3,4]]"); add_expr("attribute_tensor", "tensor<float>(a[4],b[1]):[[5],[6],[7],[8]]"); add_expr("bias_tensor", "tensor<float>(a[1],b[2]):[[4,5]]"); - add_onnx("dynamic", dynamic_model); + add_onnx(OnnxModel("dynamic", dynamic_model)); compile(onnx_feature("dynamic")); EXPECT_EQ(get(1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); EXPECT_EQ(get("onnxModel(dynamic).output", 1), TensorSpec("tensor<float>(d0[1],d1[1])").add({{"d0",0},{"d1",0}}, 79.0)); @@ -112,7 +112,7 @@ TEST_F(OnnxFeatureTest, dynamic_onnx_model_can_be_calculated) { TEST_F(OnnxFeatureTest, strange_input_and_output_names_are_normalized) { add_expr("input_0", "tensor<float>(a[2]):[10,20]"); add_expr("input_1", "tensor<float>(a[2]):[5,10]"); - add_onnx("strange_names", strange_names_model); + add_onnx(OnnxModel("strange_names", strange_names_model)); compile(onnx_feature("strange_names")); auto expect_add = TensorSpec("tensor<float>(d0[2])").add({{"d0",0}},15).add({{"d0",1}},30); auto expect_sub = TensorSpec("tensor<float>(d0[2])").add({{"d0",0}},5).add({{"d0",1}},10); @@ -121,4 +121,20 @@ TEST_F(OnnxFeatureTest, strange_input_and_output_names_are_normalized) { EXPECT_EQ(get("onnxModel(strange_names)._baz_0", 1), expect_sub); } +TEST_F(OnnxFeatureTest, input_features_and_output_names_can_be_specified) { + add_expr("my_first_input", "tensor<float>(a[2]):[10,20]"); + add_expr("my_second_input", "tensor<float>(a[2]):[5,10]"); + add_onnx(OnnxModel("custom_names", strange_names_model) + .input_feature("input:0", "rankingExpression(my_first_input)") + .input_feature("input/1", "rankingExpression(my_second_input)") + .output_name("foo/bar", "my_first_output") + .output_name("-baz:0", "my_second_output")); + compile(onnx_feature("custom_names")); + auto expect_add = TensorSpec("tensor<float>(d0[2])").add({{"d0",0}},15).add({{"d0",1}},30); + auto expect_sub = TensorSpec("tensor<float>(d0[2])").add({{"d0",0}},5).add({{"d0",1}},10); + EXPECT_EQ(get(1), expect_add); + EXPECT_EQ(get("onnxModel(custom_names).my_first_output", 1), expect_add); + EXPECT_EQ(get("onnxModel(custom_names).my_second_output", 1), expect_sub); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index 9e5021b4778..fffb70467a3 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -329,11 +329,12 @@ fillDomainTest(TransLogServer & s1, const vespalib::string & domain, size_t numP Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&value, sizeof(value))); p->add(e); if ( p->sizeBytes() > DEFAULT_PACKET_SIZE ) { - domainWriter->commit(*p, std::make_shared<CountDone>(inFlight)); + domainWriter->append(*p, std::make_shared<CountDone>(inFlight)); p = std::make_unique<Packet>(DEFAULT_PACKET_SIZE); } } - domainWriter->commit(*p, std::make_shared<CountDone>(inFlight)); + domainWriter->append(*p, std::make_shared<CountDone>(inFlight)); + auto keep = domainWriter->startCommit(Writer::DoneCallback()); LOG(info, "Inflight %ld", inFlight.load()); } while (inFlight.load() != 0) { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index 430f2eaa560..cfbca71bd5e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -114,6 +114,8 @@ to_distance_metric(const vespalib::string& metric) return DistanceMetric::Angular; } else if (metric == geodegrees) { return DistanceMetric::GeoDegrees; + } else if (metric == innerproduct) { + return DistanceMetric::InnerProduct; } else if (metric == hamming) { return DistanceMetric::Hamming; } else { diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp index 341112f9b22..4efd64f72dd 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributefilebufferwriter.h" +#include <vespa/vespalib/data/databuffer.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp index 415c00cb8fd..829720e9c3e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp index b354566616b..454cc486f70 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememoryfilebufferwriter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributememoryfilebufferwriter.h" +#include <vespa/vespalib/data/databuffer.h> namespace search { @@ -11,9 +12,7 @@ AttributeMemoryFileBufferWriter(IAttributeFileWriter &memoryFileWriter) } -AttributeMemoryFileBufferWriter::~AttributeMemoryFileBufferWriter() -{ -} +AttributeMemoryFileBufferWriter::~AttributeMemoryFileBufferWriter() = default; void diff --git a/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp index c28b7e9d20b..8d412364815 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememoryfilewriter.cpp @@ -2,6 +2,7 @@ #include "attributememoryfilewriter.h" #include "attributememoryfilebufferwriter.h" +#include <vespa/vespalib/data/databuffer.h> namespace search { @@ -18,9 +19,7 @@ AttributeMemoryFileWriter::AttributeMemoryFileWriter() } -AttributeMemoryFileWriter::~AttributeMemoryFileWriter() -{ -} +AttributeMemoryFileWriter::~AttributeMemoryFileWriter() = default; AttributeMemoryFileWriter::Buffer diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp index 59771d7ffae..80f72aaea25 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp @@ -4,6 +4,7 @@ #include "attrvector.hpp" #include "iattributesavetarget.h" #include "load_utils.h" +#include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attr_vector"); diff --git a/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h b/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h index bb00124c9fc..94e16b37e9d 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h +++ b/searchlib/src/vespa/searchlib/attribute/iattributefilewriter.h @@ -2,8 +2,9 @@ #pragma once -#include <vespa/vespalib/data/databuffer.h> +#include <memory> +namespace vespalib { class DataBuffer; } namespace search { class BufferWriter; diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index e1ab47ed434..bbe8c9c8327 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/common/bitvectoriterator.h> +#include <vespa/vespalib/data/databuffer.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp index db8636f47c3..3c26e960a06 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp @@ -2,6 +2,7 @@ #include "singlenumericattributesaver.h" #include "iattributesavetarget.h" +#include <vespa/vespalib/data/databuffer.h> using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index 6eff0da06e8..fd2631ac63d 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -7,6 +7,7 @@ #include "iattributesavetarget.h" #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/vespalib/data/databuffer.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/common/fileheadercontext.h b/searchlib/src/vespa/searchlib/common/fileheadercontext.h index 6f76fe1717d..8bb3d6a56a6 100644 --- a/searchlib/src/vespa/searchlib/common/fileheadercontext.h +++ b/searchlib/src/vespa/searchlib/common/fileheadercontext.h @@ -3,40 +3,22 @@ #include <vespa/vespalib/stllike/string.h> -namespace vespalib -{ - -class GenericHeader; - +namespace vespalib { + class GenericHeader; } -namespace search -{ - -namespace common -{ +namespace search::common { class FileHeaderContext { public: FileHeaderContext(); + virtual ~FileHeaderContext(); - virtual - ~FileHeaderContext(); + virtual void addTags(vespalib::GenericHeader &header, const vespalib::string &name) const = 0; - virtual void - addTags(vespalib::GenericHeader &header, - const vespalib::string &name) const = 0; - - static void - addCreateAndFreezeTime(vespalib::GenericHeader &header); - - static void - setFreezeTime(vespalib::GenericHeader &header); + static void addCreateAndFreezeTime(vespalib::GenericHeader &header); + static void setFreezeTime(vespalib::GenericHeader &header); }; - -} // namespace common - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def index 38741745773..defce8c3421 100644 --- a/searchlib/src/vespa/searchlib/config/translogserver.def +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -15,7 +15,7 @@ basedir string default="tmp" restart ## Use fsync after each commit. ## If not the below interval is used. -usefsync bool default=false restart +usefsync bool default=false ##Number of threads available for visiting/subscription. maxthreads int default=4 restart @@ -24,12 +24,12 @@ maxthreads int default=4 restart crcmethod enum {ccitt_crc32, xxh64} default=xxh64 ## Control compression type. -compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=LZ4 +compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=NONE ## Control compression level ## LZ4 has normal range 1..9 while ZSTD has range 1..19 ## 9 is a reasonable default for both -compression.level int default=9 +compression.level int default=3 ## How large a chunk can grow in memory before beeing flushed chunk.sizelimit int default = 256000 # 256k diff --git a/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt b/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt index 2b82d9e5af7..b1c27e20210 100644 --- a/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/docstore/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_docstore OBJECT SOURCES - bytecomplens.cpp chunk.cpp chunkformat.cpp chunkformats.cpp diff --git a/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp b/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp deleted file mode 100644 index 4ef57b77dbd..00000000000 --- a/searchlib/src/vespa/searchlib/docstore/bytecomplens.cpp +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bytecomplens.h" - -#include <vespa/log/log.h> -LOG_SETUP(".search.docstore"); - -namespace search { - -static inline uint64_t getBCN(const uint8_t *&buffer) __attribute__((__always_inline__)); - -/** - * get "Byte Compressed Number" from buffer, incrementing pointer - **/ -static inline uint64_t getBCN(const uint8_t *&buffer) -{ - uint8_t b = *buffer++; - uint64_t len = (b & 127); - unsigned shiftLen = 0; - while (b & 128) { - shiftLen += 7; - b = *buffer++; - len |= ((b & 127) << shiftLen); - } - return len; -} - -static size_t writeLen(vespalib::DataBuffer &buf, uint64_t len) -{ - size_t bytes = 0; - do { - uint8_t b = len & 127; - len >>= 7; - if (len > 0) { - b |= 128; - } - buf.writeInt8(b); - ++bytes; - } while (len > 0); - return bytes; -} - - -ByteCompressedLengths::ByteCompressedLengths() - : _l0space(), - _l1space(), - _l2space(), - _l3table(), - _entries(0), - _progress(), - _ptrcache(), - _hasInitialOffset(false) -{ - clear(); -} - - -void -ByteCompressedLengths::clear() -{ - _l0space.clear(); - _l1space.clear(); - _l2space.clear(); - _l3table.clear(); - - _entries = 0; - - _progress.lenSum1 = 0; - _progress.lenSum2 = 0; - _progress.l0oSum1 = 0; - _progress.l0oSum2 = 0; - _progress.l1oSum2 = 0; - _progress.last_offset = 0; - - _ptrcache.l0table = NULL; - _ptrcache.l1table = NULL; - _ptrcache.l2table = NULL; - - _hasInitialOffset = false; -} - - -void -ByteCompressedLengths::swap(ByteCompressedLengths& other) -{ - _l0space.swap(other._l0space); - _l1space.swap(other._l1space); - _l2space.swap(other._l2space); - _l3table.swap(other._l3table); - - std::swap(_entries, other._entries); - std::swap(_progress, other._progress); - std::swap(_ptrcache, other._ptrcache); - std::swap(_hasInitialOffset, other._hasInitialOffset); -} - - -// add a new offset to the compressed tables -void -ByteCompressedLengths::addOffset(uint64_t offset) -{ - assert(offset >= _progress.last_offset); - - // delta from last offset: - uint64_t len = offset - _progress.last_offset; - - // which entry is this: - uint64_t idx = _entries++; - - if ((idx & 31) == 0) { - // add entry to some skip-table - _progress.lenSum2 += _progress.lenSum1; // accumulate to Level2 - _progress.l0oSum2 += _progress.l0oSum1; // accumulate to Level2 - - uint64_t t1n = idx >> 5; - if ((t1n & 31) == 0) { - // add Level2 or Level3 table entry: - uint64_t t2n = t1n >> 5; - - if ((t2n & 31) == 0) { - // add new Level3 table entry: - L3Entry e; - e.offset = _progress.last_offset; - e.l0toff = _l0space.getDataLen(); - e.l1toff = _l1space.getDataLen(); - e.l2toff = _l2space.getDataLen(); - - _l3table.push_back(e); - } else { - // write to Level2 table, sums since last reset: - writeLen(_l2space, _progress.lenSum2); // sum of Level0 lengths - writeLen(_l2space, _progress.l0oSum2); // sum size of Level0 entries - writeLen(_l2space, _progress.l1oSum2); // sum size of Level1 entries - } - // reset Level2 sums: - _progress.lenSum2 = 0; - _progress.l0oSum2 = 0; - _progress.l1oSum2 = 0; - } else { - // write to Level1 table, sums since last reset: - _progress.l1oSum2 += writeLen(_l1space, _progress.lenSum1); // sum of Level0 lengths - _progress.l1oSum2 += writeLen(_l1space, _progress.l0oSum1); // sum size of Level0 entries - } - // reset Level1 sums: - _progress.lenSum1 = 0; - _progress.l0oSum1 = 0; - } - // always write length (offset delta) to Level0 table: - _progress.l0oSum1 += writeLen(_l0space, len); // accumulate to Level1 - _progress.lenSum1 += len; // accumulate to Level1 - _progress.last_offset = offset; -} - - -void -ByteCompressedLengths::addOffsetTable(uint64_t entries, uint64_t *offsets) -{ - // ignore NOP: - if (entries == 0) return; - - // Do we have some offsets already? - if (_hasInitialOffset) { - // yes, add first offset normally - addOffset(offsets[0]); - } else { - // no, special treatment for very first offset - _progress.last_offset = offsets[0]; - _hasInitialOffset = true; - } - for (uint64_t cnt = 1; cnt < entries; ++cnt) { - addOffset(offsets[cnt]); - } - - // Simplify access to actual data: - _ptrcache.l0table = (uint8_t *)_l0space.getData(); - _ptrcache.l1table = (uint8_t *)_l1space.getData(); - _ptrcache.l2table = (uint8_t *)_l2space.getData(); - - // some statistics available when debug logging: - LOG(debug, "compressed %" PRIu64 " offsets", (_entries+1)); - LOG(debug, "(%" PRIu64 " bytes)", (_entries+1)*sizeof(uint64_t)); - LOG(debug, "to (%ld + %ld + %ld) bytes + %ld l3entries", - _l0space.getDataLen(), - _l1space.getDataLen(), - _l2space.getDataLen(), - _l3table.size()); - LOG(debug, "(%ld bytes)", - (_l0space.getDataLen() + _l1space.getDataLen() + _l2space.getDataLen() + - _l3table.size()*sizeof(L3Entry))); -} - - -ByteCompressedLengths::~ByteCompressedLengths() -{ -} - -ByteCompressedLengths::OffLen -ByteCompressedLengths::getOffLen(uint64_t idx) const -{ - assert(idx < _entries); - - unsigned skipL0 = idx & 31; - unsigned skipL1 = (idx >> 5) & 31; - unsigned skipL2 = (idx >> 10) & 31; - uint64_t skipL3 = (idx >> 15); - - uint64_t offset = _l3table[skipL3].offset; - uint64_t l0toff = _l3table[skipL3].l0toff; - uint64_t l1toff = _l3table[skipL3].l1toff; - uint64_t l2toff = _l3table[skipL3].l2toff; - - // printf("start off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - - const uint8_t *l2pos = _ptrcache.l2table + l2toff; - - while (skipL2 > 0) { - --skipL2; - offset += getBCN(l2pos); - l0toff += getBCN(l2pos); - l1toff += getBCN(l2pos); - } - - const uint8_t *l1pos = _ptrcache.l1table + l1toff; - - while (skipL1 > 0) { - --skipL1; - offset += getBCN(l1pos); - l0toff += getBCN(l1pos); - - } - const uint8_t *l0pos = _ptrcache.l0table + l0toff; - - while (skipL0 > 0) { - --skipL0; - offset += getBCN(l0pos); - } - // printf("end off %ld l0off %ld l1off %ld l2off %ld\n", offset, l0toff, l1toff, l2toff); - OffLen retval; - retval.offset = offset; - retval.length = getBCN(l0pos); - return retval; -} - - -size_t -ByteCompressedLengths::memoryUsed() const -{ - size_t mem = sizeof(*this); - mem += _l0space.getBufSize(); - mem += _l1space.getBufSize(); - mem += _l2space.getBufSize(); - mem += _l3table.capacity() * sizeof(L3Entry); - return mem; -} - - - - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/docstore/bytecomplens.h b/searchlib/src/vespa/searchlib/docstore/bytecomplens.h deleted file mode 100644 index 88f6a11c764..00000000000 --- a/searchlib/src/vespa/searchlib/docstore/bytecomplens.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vector> -#include <vespa/vespalib/data/databuffer.h> - -namespace search { - -/** - * Class compressing a table of offsets in memory. - * After adding (n) offsets you can access - * (n-1) pairs of (length, offset). - * All offsets must be increasing, but they - * may be added in several chunks. - **/ -class ByteCompressedLengths -{ -public: - /** - * Construct an empty instance - **/ - ByteCompressedLengths(); - - /** - * add the given offset table. - * @param entries number of offsets to store. - * @param offsets pointer to table that contains (entries) offsets. - **/ - void addOffsetTable(uint64_t entries, uint64_t *offsets); - - /** - * free resources - **/ - ~ByteCompressedLengths(); - - struct OffLen - { - uint64_t offset; - uint64_t length; - }; - - /** - * Fetch an offset and length from compressed data. - * Note restriction: idx must be < size() - * - * @param idx The index into the offset table - * @return offset[id] and the delta (offset[id+1] - offset[id]) - **/ - OffLen getOffLen(uint64_t idx) const; - - /** - * The number of (length, offset) pairs stored - * Note that size() == sum(entries) - 1 - **/ - uint64_t size() const { return _entries; } - - /** - * remove all data from this instance - **/ - void clear(); - - /** - * swap all data with another instance - **/ - void swap(ByteCompressedLengths& other); - - /** - * Calculate memory used by this instance - * @return memory usage (in bytes) - **/ - size_t memoryUsed() const; - -private: - struct L3Entry { - uint64_t offset; - uint64_t l0toff; - uint64_t l1toff; - uint64_t l2toff; - }; - vespalib::DataBuffer _l0space; - vespalib::DataBuffer _l1space; - vespalib::DataBuffer _l2space; - - std::vector<L3Entry> _l3table; - - uint64_t _entries; - - struct ProgressPoint { - uint64_t lenSum1; - uint64_t lenSum2; - uint64_t l0oSum1; - uint64_t l0oSum2; - uint64_t l1oSum2; - uint64_t last_offset; - } _progress; - - struct CachedPointers { - const uint8_t *l0table; - const uint8_t *l1table; - const uint8_t *l2table; - } _ptrcache; - - bool _hasInitialOffset; - - void addOffset(uint64_t offset); -}; - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index fbbdcff3c5d..afa7abb9ef8 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -140,7 +140,7 @@ ChunkFormat::deserializeBody(vespalib::nbostream & is) assert(uncompressed.getData() == uncompressed.getDead()); if (uncompressed.getData() != data.c_str()) { const size_t sz(uncompressed.getDataLen()); - vespalib::nbostream(uncompressed.stealBuffer(), sz).swap(_dataBuf); + vespalib::nbostream(std::move(uncompressed).stealBuffer(), sz).swap(_dataBuf); } else { _dataBuf = vespalib::nbostream(uncompressed.getData(), uncompressed.getDataLen()); } diff --git a/searchlib/src/vespa/searchlib/docstore/value.cpp b/searchlib/src/vespa/searchlib/docstore/value.cpp index 09725b447cd..25cf93ac18b 100644 --- a/searchlib/src/vespa/searchlib/docstore/value.cpp +++ b/searchlib/src/vespa/searchlib/docstore/value.cpp @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "value.h" -#include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/compressor.h> #include <xxhash.h> @@ -66,8 +65,9 @@ Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &com _compression = type; _uncompressedSize = len; _uncompressedCrc = XXH64(input.c_str(), input.size(), 0); - _buf = std::make_shared<Alloc>(compact(_compressedSize, - (buf.getData() == compressed.getData()) ? buf.stealBuffer() : compressed.stealBuffer())); + _buf = std::make_shared<Alloc>(compact(_compressedSize,(buf.getData() == compressed.getData()) + ? std::move(buf).stealBuffer() + : std::move(compressed).stealBuffer())); assert(((type == CompressionConfig::NONE) && (len == ssize_t(_compressedSize))) || diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index e8504480b7d..6990a0a3ed7 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -109,7 +109,7 @@ CompressedBlobSet::getBlobSet() const decompress(_compression, getBufferSize(_positions), ConstBufferRef(_buffer->c_str(), _buffer->size()), uncompressed, false); } - return BlobSet(_positions, uncompressed.stealBuffer()); + return BlobSet(_positions, std::move(uncompressed).stealBuffer()); } size_t CompressedBlobSet::size() const { diff --git a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp index 698d2309e5a..fca8988ba36 100644 --- a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp @@ -2,6 +2,7 @@ #include "onnx_feature.h" #include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/onnx_model.h> #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/eval/tensor/dense/dense_tensor_view.h> #include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> @@ -85,37 +86,45 @@ OnnxBlueprint::setup(const IIndexEnvironment &env, auto optimize = (env.getFeatureMotivation() == env.FeatureMotivation::VERIFY_SETUP) ? Onnx::Optimize::DISABLE : Onnx::Optimize::ENABLE; - auto file_name = env.getOnnxModelFullPath(params[0].getValue()); - if (!file_name.has_value()) { + auto model_cfg = env.getOnnxModel(params[0].getValue()); + if (!model_cfg) { return fail("no model with name '%s' found", params[0].getValue().c_str()); } try { - _model = std::make_unique<Onnx>(file_name.value(), optimize); + _model = std::make_unique<Onnx>(model_cfg->file_path(), optimize); } catch (std::exception &ex) { return fail("model setup failed: %s", ex.what()); } Onnx::WirePlanner planner; for (size_t i = 0; i < _model->inputs().size(); ++i) { const auto &model_input = _model->inputs()[i]; - vespalib::string input_name = normalize_name(model_input.name, "input"); - if (auto maybe_input = defineInput(fmt("rankingExpression(\"%s\")", input_name.c_str()), AcceptInput::OBJECT)) { + auto input_feature = model_cfg->input_feature(model_input.name); + if (!input_feature.has_value()) { + input_feature = fmt("rankingExpression(\"%s\")", normalize_name(model_input.name, "input").c_str()); + } + if (auto maybe_input = defineInput(input_feature.value(), AcceptInput::OBJECT)) { const FeatureType &feature_input = maybe_input.value(); assert(feature_input.is_object()); if (!planner.bind_input_type(feature_input.type(), model_input)) { - return fail("incompatible type for input '%s': %s -> %s", input_name.c_str(), + return fail("incompatible type for input (%s -> %s): %s -> %s", + input_feature.value().c_str(), model_input.name.c_str(), feature_input.type().to_spec().c_str(), model_input.type_as_string().c_str()); } } } for (size_t i = 0; i < _model->outputs().size(); ++i) { const auto &model_output = _model->outputs()[i]; - vespalib::string output_name = normalize_name(model_output.name, "output"); + auto output_name = model_cfg->output_name(model_output.name); + if (!output_name.has_value()) { + output_name = normalize_name(model_output.name, "output"); + } ValueType output_type = planner.make_output_type(model_output); if (output_type.is_error()) { - return fail("unable to make compatible type for output '%s': %s -> error", - output_name.c_str(), model_output.type_as_string().c_str()); + return fail("unable to make compatible type for output (%s -> %s): %s -> error", + model_output.name.c_str(), output_name.value().c_str(), + model_output.type_as_string().c_str()); } - describeOutput(output_name, "output from onnx model", FeatureType::object(output_type)); + describeOutput(output_name.value(), "output from onnx model", FeatureType::object(output_type)); } _wire_info = planner.get_wire_info(*_model); return true; diff --git a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt index 178de1b8b87..d6f8764cd63 100644 --- a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt @@ -4,12 +4,12 @@ vespa_add_library(searchlib_fef OBJECT blueprint.cpp blueprintfactory.cpp blueprintresolver.cpp + feature_resolver.cpp feature_type.cpp featureexecutor.cpp featurenamebuilder.cpp featurenameparser.cpp featureoverrider.cpp - feature_resolver.cpp fef.cpp fieldinfo.cpp fieldpositionsiterator.cpp @@ -19,6 +19,7 @@ vespa_add_library(searchlib_fef OBJECT matchdata.cpp matchdatalayout.cpp objectstore.cpp + onnx_model.cpp parameter.cpp parameterdescriptions.cpp parametervalidator.cpp diff --git a/searchlib/src/vespa/searchlib/fef/iindexenvironment.h b/searchlib/src/vespa/searchlib/fef/iindexenvironment.h index 26e88a98033..384b81643cc 100644 --- a/searchlib/src/vespa/searchlib/fef/iindexenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iindexenvironment.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <optional> namespace vespalib::eval { struct ConstantValue; } @@ -12,6 +11,7 @@ namespace search::fef { class Properties; class FieldInfo; class ITableManager; +class OnnxModel; /** * Abstract view of index related information available to the @@ -122,9 +122,9 @@ public: virtual std::unique_ptr<vespalib::eval::ConstantValue> getConstantValue(const vespalib::string &name) const = 0; /** - * Get the full path of the file containing the given onnx model + * Get configuration for the given onnx model. **/ - virtual std::optional<vespalib::string> getOnnxModelFullPath(const vespalib::string &name) const = 0; + virtual const OnnxModel *getOnnxModel(const vespalib::string &name) const = 0; virtual uint32_t getDistributionKey() const = 0; diff --git a/searchlib/src/vespa/searchlib/fef/onnx_model.cpp b/searchlib/src/vespa/searchlib/fef/onnx_model.cpp new file mode 100644 index 00000000000..ba5adaae857 --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/onnx_model.cpp @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "onnx_model.h" +#include <tuple> + +namespace search::fef { + +OnnxModel::OnnxModel(const vespalib::string &name_in, + const vespalib::string &file_path_in) + : _name(name_in), + _file_path(file_path_in), + _input_features(), + _output_names() +{ +} + +OnnxModel & +OnnxModel::input_feature(const vespalib::string &model_input_name, const vespalib::string &input_feature) { + _input_features[model_input_name] = input_feature; + return *this; +} + +OnnxModel & +OnnxModel::output_name(const vespalib::string &model_output_name, const vespalib::string &output_name) { + _output_names[model_output_name] = output_name; + return *this; +} + +std::optional<vespalib::string> +OnnxModel::input_feature(const vespalib::string &model_input_name) const { + auto pos = _input_features.find(model_input_name); + if (pos != _input_features.end()) { + return pos->second; + } + return std::nullopt; +} + +std::optional<vespalib::string> +OnnxModel::output_name(const vespalib::string &model_output_name) const { + auto pos = _output_names.find(model_output_name); + if (pos != _output_names.end()) { + return pos->second; + } + return std::nullopt; +} + +bool +OnnxModel::operator==(const OnnxModel &rhs) const { + return (std::tie(_name, _file_path, _input_features, _output_names) == + std::tie(rhs._name, rhs._file_path, rhs._input_features, rhs._output_names)); +} + +OnnxModel::~OnnxModel() = default; + +} diff --git a/searchlib/src/vespa/searchlib/fef/onnx_model.h b/searchlib/src/vespa/searchlib/fef/onnx_model.h new file mode 100644 index 00000000000..2195a50600d --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/onnx_model.h @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <optional> +#include <map> + +namespace search::fef { + +/** + * Class containing configuration for a single onnx model setup. This + * class is used both by the IIndexEnvironment api as well as the + * OnnxModels config adapter in the search core (matching component). + **/ +class OnnxModel { +private: + vespalib::string _name; + vespalib::string _file_path; + std::map<vespalib::string,vespalib::string> _input_features; + std::map<vespalib::string,vespalib::string> _output_names; + +public: + OnnxModel(const vespalib::string &name_in, + const vespalib::string &file_path_in); + ~OnnxModel(); + + const vespalib::string &name() const { return _name; } + const vespalib::string &file_path() const { return _file_path; } + OnnxModel &input_feature(const vespalib::string &model_input_name, const vespalib::string &input_feature); + OnnxModel &output_name(const vespalib::string &model_output_name, const vespalib::string &output_name); + std::optional<vespalib::string> input_feature(const vespalib::string &model_input_name) const; + std::optional<vespalib::string> output_name(const vespalib::string &model_output_name) const; + bool operator==(const OnnxModel &rhs) const; + const std::map<vespalib::string,vespalib::string> &inspect_input_features() const { return _input_features; } + const std::map<vespalib::string,vespalib::string> &inspect_output_names() const { return _output_names; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp index 6e2e0b88fbb..d2d336dcdc8 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp @@ -54,20 +54,20 @@ IndexEnvironment::addConstantValue(const vespalib::string &name, (void) insertRes; } -std::optional<vespalib::string> -IndexEnvironment::getOnnxModelFullPath(const vespalib::string &name) const +const OnnxModel * +IndexEnvironment::getOnnxModel(const vespalib::string &name) const { auto pos = _models.find(name); if (pos != _models.end()) { - return pos->second; + return &pos->second; } - return std::nullopt; + return nullptr; } void -IndexEnvironment::addOnnxModel(const vespalib::string &name, const vespalib::string &path) +IndexEnvironment::addOnnxModel(const OnnxModel &model) { - _models[name] = path; + _models.insert_or_assign(model.name(), model); } diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h index 6602d9f8ee9..0d8d0091921 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h @@ -5,6 +5,7 @@ #include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/fef/iindexenvironment.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/onnx_model.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/tablemanager.h> #include <vespa/eval/eval/value_cache/constant_value.h> @@ -47,7 +48,7 @@ public: }; using ConstantsMap = std::map<vespalib::string, Constant>; - using ModelMap = std::map<vespalib::string, vespalib::string>; + using ModelMap = std::map<vespalib::string, OnnxModel>; IndexEnvironment(); ~IndexEnvironment(); @@ -84,8 +85,8 @@ public: vespalib::eval::ValueType type, std::unique_ptr<vespalib::eval::Value> value); - std::optional<vespalib::string> getOnnxModelFullPath(const vespalib::string &name) const override; - void addOnnxModel(const vespalib::string &name, const vespalib::string &path); + const OnnxModel *getOnnxModel(const vespalib::string &name) const override; + void addOnnxModel(const OnnxModel &model); private: IndexEnvironment(const IndexEnvironment &); // hide diff --git a/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp b/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp index a78db61429d..edb12b4bf36 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/chunks.cpp @@ -94,7 +94,7 @@ XXH64CompressedChunk::decompress(nbostream & is, uint32_t uncompressedLen) { ::decompress(_type, uncompressedLen, compressed, uncompressed, false); nbostream data(uncompressed.getData(), uncompressed.getDataLen()); deserializeEntries(data); - _backing = uncompressed.stealBuffer(); + _backing = std::move(uncompressed).stealBuffer(); is.adjustReadPos(is.size()); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index 556ebca06ec..3308f3182dc 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -121,11 +121,26 @@ Packet::add(const Packet::Entry & e) _range.to(e.serial()); } +Writer::CommitResult::CommitResult() + : _callBacks() +{} +Writer::CommitResult::CommitResult( CommitPayload commitPayLoad) + : _callBacks(std::move(commitPayLoad)) +{} + +Writer::CommitResult::~CommitResult() = default; + CommitChunk::CommitChunk(size_t reserveBytes, size_t reserveCount) : _data(reserveBytes), - _callBacks() + _callBacks(std::make_shared<Writer::DoneCallbacksList>()) +{ + _callBacks->reserve(reserveCount); +} + +CommitChunk::CommitChunk(size_t reserveBytes, Writer::CommitPayload postponed) + : _data(reserveBytes), + _callBacks(std::move(postponed)) { - _callBacks.reserve(reserveCount); } CommitChunk::~CommitChunk() = default; @@ -133,7 +148,12 @@ CommitChunk::~CommitChunk() = default; void CommitChunk::add(const Packet &packet, Writer::DoneCallback onDone) { _data.merge(packet); - _callBacks.emplace_back(std::move(onDone)); + _callBacks->emplace_back(std::move(onDone)); +} + +Writer::CommitResult +CommitChunk::createCommitResult() const { + return Writer::CommitResult(_callBacks); } } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index 7cdfad44b87..5d07d51cdf2 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -16,7 +16,7 @@ class SerialNumRange { public: SerialNumRange() : _from(0), _to(0) { } - SerialNumRange(SerialNum f) : _from(f), _to(f ? f-1 : f) { } + explicit SerialNumRange(SerialNum f) : _from(f), _to(f ? f-1 : f) { } SerialNumRange(SerialNum f, SerialNum t) : _from(f), _to(t) { } bool operator == (const SerialNumRange & b) const { return cmp(b) == 0; } bool operator < (const SerialNumRange & b) const { return cmp(b) < 0; } @@ -63,7 +63,7 @@ public: vespalib::ConstBufferRef _data; }; public: - Packet(size_t reserved) : _count(0), _range(), _buf(reserved) { } + explicit Packet(size_t reserved) : _count(0), _range(), _buf(reserved) { } Packet(const void * buf, size_t sz); void add(const Entry & data); void clear() { _buf.clear(); _count = 0; _range.from(0); _range.to(0); } @@ -84,8 +84,24 @@ int makeDirectory(const char * dir); class Writer { public: using DoneCallback = std::shared_ptr<IDestructorCallback>; + using DoneCallbacksList = std::vector<DoneCallback>; + using CommitPayload = std::shared_ptr<DoneCallbacksList>; + class CommitResult { + public: + CommitResult(); + CommitResult(CommitPayload callBacks); + CommitResult(CommitResult &&) noexcept = default; + CommitResult & operator = (CommitResult &&) noexcept = default; + CommitResult(const CommitResult &) = delete; + CommitResult & operator = (const CommitResult &) = delete; + ~CommitResult(); + size_t getNumOperations() const { return _callBacks->size(); } + private: + CommitPayload _callBacks; + }; virtual ~Writer() = default; - virtual void commit(const Packet & packet, DoneCallback done) = 0; + virtual void append(const Packet & packet, DoneCallback done) = 0; + [[nodiscard]] virtual CommitResult startCommit(DoneCallback onDone) = 0; }; class WriterFactory { @@ -106,14 +122,20 @@ public: class CommitChunk { public: CommitChunk(size_t reserveBytes, size_t reserveCount); + CommitChunk(size_t reserveBytes, Writer::CommitPayload postponed); ~CommitChunk(); + bool empty() const { return _callBacks->empty(); } void add(const Packet & packet, Writer::DoneCallback onDone); size_t sizeBytes() const { return _data.sizeBytes(); } const Packet & getPacket() const { return _data; } - size_t getNumCallBacks() const { return _callBacks.size(); } + size_t getNumCallBacks() const { return _callBacks->size(); } + Writer::CommitResult createCommitResult() const; + void setCommitDoneCallback(Writer::DoneCallback onDone) { _onCommitDone = std::move(onDone); } + Writer::CommitPayload stealCallbacks() { return std::move(_callBacks); } private: - Packet _data; - std::vector<Writer::DoneCallback> _callBacks; + Packet _data; + Writer::CommitPayload _callBacks; + Writer::DoneCallback _onCommitDone; }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 415ccafda70..bd7feec0598 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -25,17 +25,25 @@ using vespalib::makeClosure; using vespalib::makeLambdaTask; using vespalib::Monitor; using vespalib::MonitorGuard; -using search::common::FileHeaderContext; using std::runtime_error; using std::make_shared; namespace search::transactionlog { +namespace { + +std::unique_ptr<CommitChunk> +createCommitChunk(const DomainConfig &cfg) { + return std::make_unique<CommitChunk>(cfg.getChunkSizeLimit(), cfg.getChunkSizeLimit()/256); +} + +} Domain::Domain(const string &domainName, const string & baseDir, Executor & executor, const DomainConfig & cfg, const FileHeaderContext &fileHeaderContext) : _config(cfg), + _currentChunk(createCommitChunk(cfg)), _lastSerial(0), - _singleCommiter(std::make_unique<vespalib::ThreadStackExecutor>(1, 128*1024)), + _singleCommitter(std::make_unique<vespalib::ThreadStackExecutor>(1, 128 * 1024)), _executor(executor), _sessionId(1), _syncMonitor(), @@ -105,7 +113,12 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } } -Domain::~Domain() { } +Domain::~Domain() { + MonitorGuard guard(_currentChunkMonitor); + guard.broadcast(); + commitChunk(grabCurrentChunk(guard), guard); + _singleCommitter->shutdown().sync(); +} DomainInfo Domain::getDomainInfo() const @@ -311,15 +324,72 @@ Domain::optionallyRotateFile(SerialNum serialNum) { } void -Domain::commit(const Packet & packet, Writer::DoneCallback onDone) -{ - (void) onDone; +Domain::append(const Packet & packet, Writer::DoneCallback onDone) { + vespalib::MonitorGuard guard(_currentChunkMonitor); + if (_lastSerial >= packet.range().from()) { + throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", + packet.range().from(), _lastSerial)); + } else { + _lastSerial = packet.range().to(); + } + _currentChunk->add(packet, std::move(onDone)); + commitIfFull(guard); +} + +Domain::CommitResult +Domain::startCommit(DoneCallback onDone) { + vespalib::MonitorGuard guard(_currentChunkMonitor); + if ( !_currentChunk->empty() ) { + auto completed = grabCurrentChunk(guard); + completed->setCommitDoneCallback(std::move(onDone)); + CommitResult result(completed->createCommitResult()); + commitChunk(std::move(completed), guard); + return result; + } + return CommitResult(); +} + +void +Domain::commitIfFull(const vespalib::MonitorGuard &guard) { + if (_currentChunk->sizeBytes() > _config.getChunkSizeLimit()) { + auto completed = std::move(_currentChunk); + _currentChunk = std::make_unique<CommitChunk>(_config.getChunkSizeLimit(), completed->stealCallbacks()); + commitChunk(std::move(completed), guard); + } +} + +std::unique_ptr<CommitChunk> +Domain::grabCurrentChunk(const vespalib::MonitorGuard & guard) { + assert(guard.monitors(_currentChunkMonitor)); + auto chunk = std::move(_currentChunk); + _currentChunk = createCommitChunk(_config); + return chunk; +} + +void +Domain::commitChunk(std::unique_ptr<CommitChunk> chunk, const vespalib::MonitorGuard & chunkOrderGuard) { + assert(chunkOrderGuard.monitors(_currentChunkMonitor)); + _singleCommitter->execute( makeLambdaTask([this, chunk = std::move(chunk)]() mutable { + doCommit(std::move(chunk)); + })); +} + +void +Domain::doCommit(std::unique_ptr<CommitChunk> chunk) { + const Packet & packet = chunk->getPacket(); + if (packet.empty()) return; + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); DomainPart::SP dp = optionallyRotateFile(entry.serial()); dp->commit(entry.serial(), packet); + if (_config.getFSyncOnCommit()) { + dp->sync(); + } cleanSessions(); + LOG(debug, "Releasing %zu acks and %zu entries and %zu bytes.", + chunk->getNumCallBacks(), chunk->getPacket().size(), chunk->sizeBytes()); } bool diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 9adff564cc8..041ec27cf23 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -18,8 +18,9 @@ public: using SP = std::shared_ptr<Domain>; using Executor = vespalib::SyncableThreadExecutor; using DomainPartSP = std::shared_ptr<DomainPart>; + using FileHeaderContext = common::FileHeaderContext; Domain(const vespalib::string &name, const vespalib::string &baseDir, Executor & executor, - const DomainConfig & cfg, const common::FileHeaderContext &fileHeaderContext); + const DomainConfig & cfg, const FileHeaderContext &fileHeaderContext); ~Domain() override; @@ -27,14 +28,14 @@ public: const vespalib::string & name() const { return _name; } bool erase(SerialNum to); - void commit(const Packet & packet, Writer::DoneCallback onDone) override; + void append(const Packet & packet, Writer::DoneCallback onDone) override; + [[nodiscard]] CommitResult startCommit(DoneCallback onDone) override; int visit(const Domain::SP & self, SerialNum from, SerialNum to, std::unique_ptr<Destination> dest); SerialNum begin() const; SerialNum end() const; SerialNum getSynced() const; void triggerSyncNow(); - bool commitIfStale(); bool getMarkedDeleted() const { return _markedDeleted; } void markDeleted() { _markedDeleted = true; } @@ -55,6 +56,11 @@ public: uint64_t size() const; Domain & setConfig(const DomainConfig & cfg); private: + void commitIfFull(const vespalib::MonitorGuard & guard); + + std::unique_ptr<CommitChunk> grabCurrentChunk(const vespalib::MonitorGuard & guard); + void commitChunk(std::unique_ptr<CommitChunk> chunk, const vespalib::MonitorGuard & chunkOrderGuard); + void doCommit(std::unique_ptr<CommitChunk> chunk); SerialNum begin(const vespalib::LockGuard & guard) const; SerialNum end(const vespalib::LockGuard & guard) const; size_t byteSize(const vespalib::LockGuard & guard) const; @@ -72,23 +78,24 @@ private: using DomainPartList = std::map<SerialNum, DomainPartSP>; using DurationSeconds = std::chrono::duration<double>; - DomainConfig _config; - SerialNum _lastSerial; - std::unique_ptr<Executor> _singleCommiter; - Executor & _executor; - std::atomic<int> _sessionId; - vespalib::Monitor _syncMonitor; - bool _pendingSync; - vespalib::string _name; - DomainPartList _parts; - vespalib::Lock _lock; - vespalib::Monitor _currentChunkMonitor; - vespalib::Lock _sessionLock; - SessionList _sessions; - DurationSeconds _maxSessionRunTime; - vespalib::string _baseDir; - const common::FileHeaderContext &_fileHeaderContext; - bool _markedDeleted; + DomainConfig _config; + std::unique_ptr<CommitChunk> _currentChunk; + SerialNum _lastSerial; + std::unique_ptr<Executor> _singleCommitter; + Executor &_executor; + std::atomic<int> _sessionId; + vespalib::Monitor _syncMonitor; + bool _pendingSync; + vespalib::string _name; + DomainPartList _parts; + vespalib::Lock _lock; + vespalib::Monitor _currentChunkMonitor; + vespalib::Lock _sessionLock; + SessionList _sessions; + DurationSeconds _maxSessionRunTime; + vespalib::string _baseDir; + const FileHeaderContext &_fileHeaderContext; + bool _markedDeleted; }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 8855183226d..b7e02894e6b 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -252,7 +252,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, Encoding encoding, uint8_t compressionLevel, const FileHeaderContext &fileHeaderContext, bool allowTruncate) - : _encoding(encoding.getCrc(), Encoding::Compression::none), //TODO We do not yet support compression + : _encoding(encoding), _compressionLevel(compressionLevel), _lock(), _fileLock(), @@ -396,16 +396,19 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) if (_range.from() == 0) { _range.from(firstSerial); } + IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); for (size_t i(0); h.size() > 0; i++) { //LOG(spam, //"Pos(%d) Len(%d), Lim(%d), Remaining(%d)", //h.getPos(), h.getLength(), h.getLimit(), h.getRemaining()); - IChunk::UP chunk = IChunk::create(_encoding, _compressionLevel); Packet::Entry entry; entry.deserialize(h); if (_range.to() < entry.serial()) { chunk->add(entry); - write(*_transLog, *chunk); + if (_encoding.getCompression() == Encoding::Compression::none) { + write(*_transLog, *chunk); + chunk = IChunk::create(_encoding, _compressionLevel); + } _sz++; _range.to(entry.serial()); } else { @@ -413,6 +416,9 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet) entry.serial(), _range.to())); } } + if ( ! chunk->getEntries().empty()) { + write(*_transLog, *chunk); + } bool merged(false); LockGuard guard(_lock); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 269ed7e9380..0c0c9186e12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -572,7 +572,11 @@ TransLogServer::domainCommit(FRT_RPCRequest *req) Packet packet(params[1]._data._buf, params[1]._data._len); try { vespalib::Gate gate; - domain->commit(packet, make_shared<GateCallback>(gate)); + { + // Need to scope in order to drain out all the callbacks. + domain->append(packet, make_shared<GateCallback>(gate)); + auto keep = domain->startCommit(make_shared<IgnoreCallback>()); + } gate.await(); ret.AddInt32(0); ret.AddString("ok"); diff --git a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index 09c9ddb1f72..8b009e02f28 100644 --- a/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -82,8 +82,7 @@ struct SetStateFixture : FixtureBase { auto* params = bound_request->GetParams(); params->AddInt8(static_cast<uint8_t>(encoded_bundle._compression_type)); params->AddInt32(uncompressed_length); - const auto buf_len = encoded_bundle._buffer->getDataLen(); - params->AddData(encoded_bundle._buffer->stealBuffer(), buf_len); + params->AddData(std::move(*encoded_bundle._buffer)); bound_request->SetDetachedPT(&request_is_detached); bound_request->SetReturnHandler(&return_handler); diff --git a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp index 69259ee08ec..ee70f265297 100644 --- a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp +++ b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp @@ -96,18 +96,18 @@ vespalib::string to_slobrok_id(const api::StorageMessageAddress& address) { return CachingRpcTargetResolver::address_to_slobrok_id(address); } -class StorageApiNode { +class RpcNode { +protected: vdstestlib::DirConfig _config; std::shared_ptr<const document::DocumentTypeRepo> _doc_type_repo; std::shared_ptr<const documentapi::LoadTypeSet> _load_type_set; LockingMockOperationDispatcher _messages; std::unique_ptr<MessageCodecProvider> _codec_provider; std::unique_ptr<SharedRpcResources> _shared_rpc_resources; - std::unique_ptr<StorageApiRpcService> _service; api::StorageMessageAddress _node_address; vespalib::string _slobrok_id; public: - StorageApiNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) + RpcNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) : _config(getStandardConfig(true)), _doc_type_repo(document::TestDocRepo().getTypeRepoSp()), _load_type_set(std::make_shared<documentapi::LoadTypeSet>()), @@ -122,13 +122,10 @@ public: _shared_rpc_resources = std::make_unique<SharedRpcResources>(_config.getConfigId(), 0, 1); // TODO make codec provider into interface so we can test decode-failures more easily? _codec_provider = std::make_unique<MessageCodecProvider>(_doc_type_repo, _load_type_set); - StorageApiRpcService::Params params; - _service = std::make_unique<StorageApiRpcService>(_messages, *_shared_rpc_resources, *_codec_provider, params); - - _shared_rpc_resources->start_server_and_register_slobrok(_slobrok_id); - // Explicitly wait until we are visible in Slobrok. Just waiting for mirror readiness is not enough. - wait_until_visible_in_slobrok(_slobrok_id); } + ~RpcNode(); + + const api::StorageMessageAddress& node_address() const noexcept { return _node_address; } void wait_until_visible_in_slobrok(vespalib::stringref id) { const auto deadline = std::chrono::steady_clock::now() + slobrok_register_timeout; @@ -139,8 +136,24 @@ public: std::this_thread::sleep_for(10ms); } } +}; - const api::StorageMessageAddress& node_address() const noexcept { return _node_address; } +RpcNode::~RpcNode() = default; + +class StorageApiNode : public RpcNode { + std::unique_ptr<StorageApiRpcService> _service; +public: + StorageApiNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) + : RpcNode(node_index, is_distributor, slobrok) + { + StorageApiRpcService::Params params; + _service = std::make_unique<StorageApiRpcService>(_messages, *_shared_rpc_resources, *_codec_provider, params); + + _shared_rpc_resources->start_server_and_register_slobrok(_slobrok_id); + // Explicitly wait until we are visible in Slobrok. Just waiting for mirror readiness is not enough. + wait_until_visible_in_slobrok(_slobrok_id); + } + ~StorageApiNode(); std::shared_ptr<api::PutCommand> create_dummy_put_command() const { auto doc_type = _doc_type_repo->getDocumentType("testdoctype1"); @@ -177,6 +190,26 @@ public: _messages.wait_until_n_messages_received(1); return _messages.pop_first_message(); } + + bool target_supports_direct_rpc(const api::StorageMessageAddress& addr) const noexcept { + return _service->target_supports_direct_rpc(addr); + } +}; + +StorageApiNode::~StorageApiNode() { + // Ensure we shut down the underlying RPC threads before destroying + // the RPC service that may receive callbacks from it. + _shared_rpc_resources->shutdown(); +} + +struct NodeWithoutStorageApiService : RpcNode { + NodeWithoutStorageApiService(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) + : RpcNode(node_index, is_distributor, slobrok) + { + _shared_rpc_resources->start_server_and_register_slobrok(_slobrok_id); + // Explicitly wait until we are visible in Slobrok. Just waiting for mirror readiness is not enough. + wait_until_visible_in_slobrok(_slobrok_id); + } }; } // anonymous namespace @@ -304,4 +337,26 @@ TEST_F(StorageApiRpcServiceTest, response_trace_only_propagated_if_trace_level_s EXPECT_THAT(trace_str, Not(HasSubstr("Doing cool things"))); } +TEST_F(StorageApiRpcServiceTest, rpc_method_not_found_toggles_rpc_as_not_supported) { + NodeWithoutStorageApiService dummy_node(10, false, _slobrok); + _node_0->wait_until_visible_in_slobrok(to_slobrok_id(dummy_node.node_address())); + + // Initially we assume targets are on a new enough version to understand storage API RPCs. + EXPECT_TRUE(_node_0->target_supports_direct_rpc(dummy_node.node_address())); + EXPECT_TRUE(_node_0->target_supports_direct_rpc(_node_1->node_address())); + + // Send to an endpoint exposing RPC but not the Storage API server method. + // It will bounce back immediately with an FRT "no such method" error. + auto cmd = _node_0->create_dummy_put_command(); + cmd->setAddress(dummy_node.node_address()); + _node_0->send_request(cmd); + auto bounced_msg = _node_0->wait_and_receive_single_message(); + ASSERT_TRUE(bounced_msg); + + // For now (and for the sake of simplicity), fall back to assuming no targets + // support direct storage API RPC. + EXPECT_FALSE(_node_0->target_supports_direct_rpc(dummy_node.node_address())); + EXPECT_FALSE(_node_0->target_supports_direct_rpc(_node_1->node_address())); +} + } diff --git a/storage/src/vespa/storage/common/CMakeLists.txt b/storage/src/vespa/storage/common/CMakeLists.txt index 20b73987bf3..81c6486eaeb 100644 --- a/storage/src/vespa/storage/common/CMakeLists.txt +++ b/storage/src/vespa/storage/common/CMakeLists.txt @@ -15,5 +15,6 @@ vespa_add_library(storage_common OBJECT storagecomponent.cpp storagelink.cpp storagelinkqueued.cpp + storage_chain_builder.cpp DEPENDS ) diff --git a/storage/src/vespa/storage/common/i_storage_chain_builder.h b/storage/src/vespa/storage/common/i_storage_chain_builder.h new file mode 100644 index 00000000000..8f4708ad7ac --- /dev/null +++ b/storage/src/vespa/storage/common/i_storage_chain_builder.h @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace storage { + +class StorageLink; + +/* + * Interface class for building a storage chain. + */ +class IStorageChainBuilder +{ +public: + virtual ~IStorageChainBuilder() = default; + virtual void add(std::unique_ptr<StorageLink> link) = 0; + virtual std::unique_ptr<StorageLink> build() && = 0; +}; + +} diff --git a/storage/src/vespa/storage/common/storage_chain_builder.cpp b/storage/src/vespa/storage/common/storage_chain_builder.cpp new file mode 100644 index 00000000000..45878f452cb --- /dev/null +++ b/storage/src/vespa/storage/common/storage_chain_builder.cpp @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_chain_builder.h" +#include "storagelink.h" + +namespace storage { + +StorageChainBuilder::StorageChainBuilder() + : _top() +{ +} + +StorageChainBuilder::~StorageChainBuilder() = default; + +void +StorageChainBuilder::add(std::unique_ptr<StorageLink> link) +{ + if (_top) { + _top->push_back(std::move(link)); + } else { + _top = std::move(link); + } +}; + +std::unique_ptr<StorageLink> +StorageChainBuilder::build() && +{ + return std::move(_top); +} + +} diff --git a/storage/src/vespa/storage/common/storage_chain_builder.h b/storage/src/vespa/storage/common/storage_chain_builder.h new file mode 100644 index 00000000000..ce4087e0bd0 --- /dev/null +++ b/storage/src/vespa/storage/common/storage_chain_builder.h @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_storage_chain_builder.h" + +namespace storage { + +/** + * Class for building a storage chain. + */ +class StorageChainBuilder : public IStorageChainBuilder +{ +protected: + std::unique_ptr<StorageLink> _top; +public: + StorageChainBuilder(); + ~StorageChainBuilder() override; + void add(std::unique_ptr<StorageLink> link) override; + std::unique_ptr<StorageLink> build() && override; +}; + +} diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 5471d66a864..667c577645a 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -575,13 +575,13 @@ CommunicationManager::sendCommand( case api::StorageMessageAddress::STORAGE: { LOG(debug, "Send to %s: %s", address.toString().c_str(), msg->toString().c_str()); - if (_use_direct_storageapi_rpc) { + if (_use_direct_storageapi_rpc && _storage_api_rpc_service->target_supports_direct_rpc(address)) { _storage_api_rpc_service->send_rpc_v1_request(msg); } else { auto cmd = std::make_unique<mbusprot::StorageCommand>(msg); cmd->setContext(mbus::Context(msg->getMsgId())); - cmd->setRetryEnabled(address.retryEnabled()); + cmd->setRetryEnabled(false); cmd->setTimeRemaining(msg->getTimeout()); cmd->setTrace(msg->getTrace()); sendMessageBusMessage(msg, std::move(cmd), address.getRoute()); @@ -597,7 +597,7 @@ CommunicationManager::sendCommand( if (mbusMsg) { MBUS_TRACE(msg->getTrace(), 7, "Communication manager: Converted OK"); mbusMsg->setTrace(msg->getTrace()); - mbusMsg->setRetryEnabled(address.retryEnabled()); + mbusMsg->setRetryEnabled(false); { vespalib::LockGuard lock(_messageBusSentLock); diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index 1cd1477e769..3d1f9bbaf2e 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -5,6 +5,7 @@ #include "communicationmanager.h" #include "opslogger.h" #include "statemanager.h" +#include <vespa/storage/common/i_storage_chain_builder.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/vespalib/util/exceptions.h> @@ -84,34 +85,33 @@ DistributorNode::handleConfigChange(vespa::config::content::core::StorVisitordis _context.getComponentRegister().setVisitorConfig(c); } -StorageLink::UP -DistributorNode::createChain() +void +DistributorNode::createChain(IStorageChainBuilder &builder) { DistributorComponentRegister& dcr(_context.getComponentRegister()); // TODO: All components in this chain should use a common thread instead of // each having its own configfetcher. StorageLink::UP chain; if (_retrievedCommunicationManager.get()) { - chain = std::move(_retrievedCommunicationManager); + builder.add(std::move(_retrievedCommunicationManager)); } else { - chain.reset(_communicationManager - = new CommunicationManager(dcr, _configUri)); + auto communication_manager = std::make_unique<CommunicationManager>(dcr, _configUri); + _communicationManager = communication_manager.get(); + builder.add(std::move(communication_manager)); } std::unique_ptr<StateManager> stateManager(releaseStateManager()); - chain->push_back(StorageLink::UP(new Bouncer(dcr, _configUri))); - chain->push_back(StorageLink::UP(new OpsLogger(dcr, _configUri))); + builder.add(std::make_unique<Bouncer>(dcr, _configUri)); + builder.add(std::make_unique<OpsLogger>(dcr, _configUri)); // Distributor instance registers a host info reporter with the state // manager, which is safe since the lifetime of said state manager // extends to the end of the process. - chain->push_back(StorageLink::UP( - new storage::distributor::Distributor( - dcr, *_threadPool, getDoneInitializeHandler(), - _manageActiveBucketCopies, - stateManager->getHostInfo()))); - - chain->push_back(StorageLink::UP(stateManager.release())); - return chain; + builder.add(std::make_unique<storage::distributor::Distributor> + (dcr, *_threadPool, getDoneInitializeHandler(), + _manageActiveBucketCopies, + stateManager->getHostInfo())); + + builder.add(std::move(stateManager)); } api::Timestamp diff --git a/storage/src/vespa/storage/storageserver/distributornode.h b/storage/src/vespa/storage/storageserver/distributornode.h index 4db8876dc24..39614674bb5 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.h +++ b/storage/src/vespa/storage/storageserver/distributornode.h @@ -49,7 +49,7 @@ public: private: void initializeNodeSpecific() override; - std::unique_ptr<StorageLink> createChain() override; + void createChain(IStorageChainBuilder &builder) override; api::Timestamp getUniqueTimestamp() override; /** diff --git a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp index 5241ec6f769..6bcb154aed5 100644 --- a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp @@ -33,9 +33,9 @@ CachingRpcTargetResolver::address_to_slobrok_id(const api::StorageMessageAddress } std::shared_ptr<RpcTarget> -CachingRpcTargetResolver::lookup_target(const vespalib::string& slobrok_id, uint32_t curr_slobrok_gen) { +CachingRpcTargetResolver::lookup_target(const api::StorageMessageAddress& address, uint32_t curr_slobrok_gen) { std::shared_lock lock(_targets_rwmutex); - auto itr = _targets.find(slobrok_id); + auto itr = _targets.find(address); if ((itr != _targets.end()) && itr->second->_target->is_valid() && (itr->second->_slobrok_gen == curr_slobrok_gen)) { @@ -45,18 +45,19 @@ CachingRpcTargetResolver::lookup_target(const vespalib::string& slobrok_id, uint } std::shared_ptr<RpcTarget> -CachingRpcTargetResolver::consider_update_target(const vespalib::string& slobrok_id, +CachingRpcTargetResolver::consider_update_target(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, [[maybe_unused]] const UniqueLock& targets_lock) { // If address has the same spec as the existing target, just reuse it. - auto itr = _targets.find(slobrok_id); + auto itr = _targets.find(address); if ((itr != _targets.end()) && (itr->second->_target->is_valid()) && (itr->second->_spec == connection_spec)) { LOG(debug, "Updating existing mapping '%s' -> '%s' (gen %u) to gen %u", - slobrok_id.c_str(), connection_spec.c_str(), itr->second->_slobrok_gen, curr_slobrok_gen); + address.toString().c_str(), connection_spec.c_str(), + itr->second->_slobrok_gen, curr_slobrok_gen); itr->second->_slobrok_gen = curr_slobrok_gen; return itr->second; } @@ -64,26 +65,27 @@ CachingRpcTargetResolver::consider_update_target(const vespalib::string& slobrok } std::shared_ptr<RpcTarget> -CachingRpcTargetResolver::insert_new_target_mapping(const vespalib::string& slobrok_id, +CachingRpcTargetResolver::insert_new_target_mapping(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, [[maybe_unused]] const UniqueLock& targets_lock) { auto target = _target_factory.make_target(connection_spec, curr_slobrok_gen); // TODO expensive inside lock? assert(target); std::shared_ptr<RpcTarget> rpc_target(std::move(target)); - _targets[slobrok_id] = rpc_target; - LOG(debug, "Added mapping '%s' -> '%s' at gen %u", slobrok_id.c_str(), connection_spec.c_str(), curr_slobrok_gen); + // TODO emplacement (with replace) semantics to avoid need for default constructed K/V + _targets[address] = rpc_target; + LOG(debug, "Added mapping '%s' -> '%s' at gen %u", address.toString().c_str(), + connection_spec.c_str(), curr_slobrok_gen); return rpc_target; } std::shared_ptr<RpcTarget> CachingRpcTargetResolver::resolve_rpc_target(const api::StorageMessageAddress& address) { - // TODO or map directly from address to target instead of going via stringification? Needs hashing, if so. - auto slobrok_id = address_to_slobrok_id(address); const uint32_t curr_slobrok_gen = _slobrok_mirror.updates(); - if (auto result = lookup_target(slobrok_id, curr_slobrok_gen)) { + if (auto result = lookup_target(address, curr_slobrok_gen)) { return result; } + auto slobrok_id = address_to_slobrok_id(address); auto specs = _slobrok_mirror.lookup(slobrok_id); // FIXME string type mismatch; implicit conv! if (specs.empty()) { LOG(debug, "Found no mapping for '%s'", slobrok_id.c_str()); @@ -95,10 +97,10 @@ CachingRpcTargetResolver::resolve_rpc_target(const api::StorageMessageAddress& a assert(specs.size() == 1); const auto& connection_spec = specs[0].second; std::unique_lock lock(_targets_rwmutex); - if (auto result = consider_update_target(slobrok_id, connection_spec, curr_slobrok_gen, lock)) { + if (auto result = consider_update_target(address, connection_spec, curr_slobrok_gen, lock)) { return result; } - return insert_new_target_mapping(slobrok_id, connection_spec, curr_slobrok_gen, lock); + return insert_new_target_mapping(address, connection_spec, curr_slobrok_gen, lock); } } diff --git a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h index cf94f7545bc..52b505d5476 100644 --- a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h +++ b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h @@ -3,38 +3,44 @@ #include "rpc_target.h" #include "rpc_target_factory.h" +#include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/vespalib/stllike/hash_map.h> #include <memory> #include <shared_mutex> namespace slobrok::api { class IMirrorAPI; } -namespace storage { - -namespace api { class StorageMessageAddress; } - -namespace rpc { +namespace storage::rpc { /** * Class that resolves and caches rpc targets based on StorageMessageAddress that is mapped to slobrok id, * with lookup in a slobrok mirror. */ class CachingRpcTargetResolver { -private: - const slobrok::api::IMirrorAPI& _slobrok_mirror; - const RpcTargetFactory& _target_factory; + + struct AddressInternalHasher { + size_t operator()(const api::StorageMessageAddress& addr) const noexcept { + return addr.internal_storage_hash(); + } + }; + using TargetHashMap = vespalib::hash_map<api::StorageMessageAddress, + std::shared_ptr<RpcTarget>, + AddressInternalHasher>; using UniqueLock = std::unique_lock<std::shared_mutex>; - mutable std::shared_mutex _targets_rwmutex; - // TODO LRU? Size cap? - vespalib::hash_map<vespalib::string, std::shared_ptr<RpcTarget>> _targets; - std::shared_ptr<RpcTarget> lookup_target(const vespalib::string& slobrok_id, uint32_t curr_slobrok_gen); - std::shared_ptr<RpcTarget> consider_update_target(const vespalib::string& slobrok_id, + const slobrok::api::IMirrorAPI& _slobrok_mirror; + const RpcTargetFactory& _target_factory; + mutable std::shared_mutex _targets_rwmutex; + TargetHashMap _targets; // TODO LRU? Size cap? + + std::shared_ptr<RpcTarget> lookup_target(const api::StorageMessageAddress& address, + uint32_t curr_slobrok_gen); + std::shared_ptr<RpcTarget> consider_update_target(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, const UniqueLock& targets_lock); - std::shared_ptr<RpcTarget> insert_new_target_mapping(const vespalib::string& slobrok_id, + std::shared_ptr<RpcTarget> insert_new_target_mapping(const api::StorageMessageAddress& address, const vespalib::string& connection_spec, uint32_t curr_slobrok_gen, const UniqueLock& targets_lock); @@ -49,5 +55,4 @@ public: std::shared_ptr<RpcTarget> resolve_rpc_target(const api::StorageMessageAddress& address); }; -} // rpc -} // storage +} // storage::rpc diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp index e34030f9bd7..4c7cf05241a 100644 --- a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp @@ -20,7 +20,6 @@ ClusterControllerApiRpcService::ClusterControllerApiRpcService( MessageDispatcher& message_dispatcher, SharedRpcResources& rpc_resources) : _message_dispatcher(message_dispatcher), - _rpc_resources(rpc_resources), _closed(false) { register_server_methods(rpc_resources); diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h index 793644194dc..d5e753ff04b 100644 --- a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h @@ -24,7 +24,6 @@ class SharedRpcResources; class ClusterControllerApiRpcService : public FRT_Invokable { MessageDispatcher& _message_dispatcher; - SharedRpcResources& _rpc_resources; std::atomic<bool> _closed; public: static constexpr uint32_t StateBundleMaxUncompressedSize = 1024 * 1024 * 16; diff --git a/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h b/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h index 8411a273dc2..ef438e96305 100644 --- a/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h +++ b/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h @@ -6,7 +6,7 @@ namespace storage::rpc { -class RpcTarget; +struct RpcTarget; /** * Factory for creating instances of RpcTarget based on a connection spec. diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index 68339e9c493..c02f0c56092 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -33,7 +33,8 @@ StorageApiRpcService::StorageApiRpcService(MessageDispatcher& message_dispatcher _rpc_resources(rpc_resources), _message_codec_provider(message_codec_provider), _params(params), - _target_resolver(std::make_unique<CachingRpcTargetResolver>(_rpc_resources.slobrok_mirror(), _rpc_resources.target_factory())) + _target_resolver(std::make_unique<CachingRpcTargetResolver>(_rpc_resources.slobrok_mirror(), _rpc_resources.target_factory())), + _direct_rpc_supported(true) { register_server_methods(rpc_resources); } @@ -117,7 +118,7 @@ void compress_and_add_payload_to_rpc_params(mbus::BlobRef payload, params.AddInt8(comp_type); params.AddInt32(static_cast<uint32_t>(to_compress.size())); - params.AddData(buf.stealBuffer(), buf.getDataLen()); + params.AddData(std::move(buf)); } } // anon ns @@ -225,12 +226,7 @@ void StorageApiRpcService::RequestDone(FRT_RPCRequest* raw_req) { std::unique_ptr<FRT_RPCRequest, SubRefDeleter> req(raw_req); auto* req_ctx = static_cast<RpcRequestContext*>(req->GetContext()._value.VOIDP); if (!req->CheckReturnTypes("bixbix")) { - api::ReturnCode error = map_frt_error_to_storage_api_error(*req, *req_ctx); - LOG(debug, "Client: received rpc.v1 error response: %s", error.toString().c_str()); - auto error_reply = req_ctx->_originator_cmd->makeReply(); - error_reply->setResult(std::move(error)); - // TODO needs tracing of received-event! - _message_dispatcher.dispatch_sync(std::move(error_reply)); + handle_request_done_rpc_error(*req, *req_ctx); return; } LOG(debug, "Client: received rpc.v1 OK response"); @@ -259,6 +255,22 @@ void StorageApiRpcService::RequestDone(FRT_RPCRequest* raw_req) { _message_dispatcher.dispatch_sync(std::move(reply)); } +void StorageApiRpcService::handle_request_done_rpc_error(FRT_RPCRequest& req, + const RpcRequestContext& req_ctx) { + auto error_reply = req_ctx._originator_cmd->makeReply(); + api::ReturnCode error; + if (req.GetErrorCode() == FRTE_RPC_NO_SUCH_METHOD) { + mark_peer_without_direct_rpc_support(*req_ctx._originator_cmd->getAddress()); + error = api::ReturnCode(api::ReturnCode::NOT_CONNECTED, "Direct Storage RPC protocol not supported"); + } else { + error = map_frt_error_to_storage_api_error(req, req_ctx); + } + LOG(debug, "Client: received rpc.v1 error response: %s", error.toString().c_str()); + error_reply->setResult(std::move(error)); + // TODO needs tracing of received-event! + _message_dispatcher.dispatch_sync(std::move(error_reply)); +} + api::ReturnCode StorageApiRpcService::map_frt_error_to_storage_api_error(FRT_RPCRequest& req, const RpcRequestContext& req_ctx) { @@ -297,6 +309,23 @@ StorageApiRpcService::make_no_address_for_service_error(const api::StorageMessag return api::ReturnCode(error_code, std::move(error_msg)); } +void StorageApiRpcService::mark_peer_without_direct_rpc_support(const api::StorageMessageAddress& addr) { + bool expected = true; + if (_direct_rpc_supported.compare_exchange_strong(expected, false, std::memory_order_relaxed)) { + LOG(info, "Node %s does not support direct Storage API RPC; falling back " + "to legacy MessageBus protocol. Not logging this for any further nodes", + addr.toString().c_str()); + } +} + +bool StorageApiRpcService::target_supports_direct_rpc( + [[maybe_unused]] const api::StorageMessageAddress& addr) const noexcept { + // Stale reads isn't an issue here, since the worst case is just receiving + // a few more "no such method" errors. + return _direct_rpc_supported.load(std::memory_order_relaxed); +} + + /* * Major TODOs: * - tracing and trace propagation diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h index 3fca08acc15..c8152ebfdbd 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h @@ -47,6 +47,7 @@ private: MessageCodecProvider& _message_codec_provider; const Params _params; std::unique_ptr<CachingRpcTargetResolver> _target_resolver; + std::atomic<bool> _direct_rpc_supported; public: StorageApiRpcService(MessageDispatcher& message_dispatcher, SharedRpcResources& rpc_resources, @@ -54,6 +55,8 @@ public: const Params& params); ~StorageApiRpcService() override; + [[nodiscard]] bool target_supports_direct_rpc(const api::StorageMessageAddress& addr) const noexcept; + void RPC_rpc_v1_send(FRT_RPCRequest* req); void encode_rpc_v1_response(FRT_RPCRequest& request, const api::StorageReply& reply); void send_rpc_v1_request(std::shared_ptr<api::StorageCommand> cmd); @@ -76,8 +79,12 @@ private: void encode_and_compress_rpc_payload(const MessageType& msg, FRT_Values& params); void RequestDone(FRT_RPCRequest* request) override; + void handle_request_done_rpc_error(FRT_RPCRequest& req, const RpcRequestContext& req_ctx); + api::ReturnCode map_frt_error_to_storage_api_error(FRT_RPCRequest& req, const RpcRequestContext& req_ctx); api::ReturnCode make_no_address_for_service_error(const api::StorageMessageAddress& addr) const; + + void mark_peer_without_direct_rpc_support(const api::StorageMessageAddress& addr); }; } // rpc diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index 1b90e235d20..4f1db0e1b30 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -9,6 +9,7 @@ #include "statemanager.h" #include "priorityconverter.h" #include "service_layer_error_listener.h" +#include <vespa/storage/common/i_storage_chain_builder.h> #include <vespa/storage/visiting/messagebusvisitormessagesession.h> #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storage/bucketdb/bucketmanager.h> @@ -220,43 +221,44 @@ ServiceLayerNode::toDocumentPriority(uint8_t storagePriority) const return _communicationManager->getPriorityConverter().toDocumentPriority(storagePriority); } -StorageLink::UP -ServiceLayerNode::createChain() +void +ServiceLayerNode::createChain(IStorageChainBuilder &builder) { ServiceLayerComponentRegister& compReg(_context.getComponentRegister()); - StorageLink::UP chain; - chain.reset(_communicationManager = new CommunicationManager(compReg, _configUri)); - chain->push_back(StorageLink::UP(new Bouncer(compReg, _configUri))); + auto communication_manager = std::make_unique<CommunicationManager>(compReg, _configUri); + _communicationManager = communication_manager.get(); + builder.add(std::move(communication_manager)); + builder.add(std::make_unique<Bouncer>(compReg, _configUri)); if (_noUsablePartitionMode) { /* * No usable partitions. Use minimal chain. Still needs to be * able to report state back to cluster controller. */ - chain->push_back(StorageLink::UP(releaseStateManager().release())); - return chain; + builder.add(releaseStateManager()); + return; } - chain->push_back(StorageLink::UP(new OpsLogger(compReg, _configUri))); - auto* merge_throttler = new MergeThrottler(_configUri, compReg); - chain->push_back(StorageLink::UP(merge_throttler)); - chain->push_back(StorageLink::UP(new ChangedBucketOwnershipHandler(_configUri, compReg))); - chain->push_back(StorageLink::UP(new StorageBucketDBInitializer( - _configUri, _partitions, getDoneInitializeHandler(), compReg))); - chain->push_back(StorageLink::UP(new BucketManager(_configUri, _context.getComponentRegister()))); - chain->push_back(StorageLink::UP(new VisitorManager( - _configUri, _context.getComponentRegister(), *this, _externalVisitors))); - chain->push_back(StorageLink::UP(new ModifiedBucketChecker( - _context.getComponentRegister(), _persistenceProvider, _configUri))); - chain->push_back(StorageLink::UP(_fileStorManager = new FileStorManager( - _configUri, _partitions, _persistenceProvider, _context.getComponentRegister()))); - chain->push_back(StorageLink::UP(releaseStateManager().release())); + builder.add(std::make_unique<OpsLogger>(compReg, _configUri)); + auto merge_throttler_up = std::make_unique<MergeThrottler>(_configUri, compReg); + auto merge_throttler = merge_throttler_up.get(); + builder.add(std::move(merge_throttler_up)); + builder.add(std::make_unique<ChangedBucketOwnershipHandler>(_configUri, compReg)); + builder.add(std::make_unique<StorageBucketDBInitializer>( + _configUri, _partitions, getDoneInitializeHandler(), compReg)); + builder.add(std::make_unique<BucketManager>(_configUri, _context.getComponentRegister())); + builder.add(std::make_unique<VisitorManager>(_configUri, _context.getComponentRegister(), static_cast<VisitorMessageSessionFactory &>(*this), _externalVisitors)); + builder.add(std::make_unique<ModifiedBucketChecker>( + _context.getComponentRegister(), _persistenceProvider, _configUri)); + auto filstor_manager = std::make_unique<FileStorManager>(_configUri, _partitions, _persistenceProvider, _context.getComponentRegister()); + _fileStorManager = filstor_manager.get(); + builder.add(std::move(filstor_manager)); + builder.add(releaseStateManager()); // Lifetimes of all referenced components shall outlive the last call going // through the SPI, as queues are flushed and worker threads joined when // the storage link chain is closed prior to destruction. auto error_listener = std::make_shared<ServiceLayerErrorListener>(*_component, *merge_throttler); _fileStorManager->error_wrapper().register_error_listener(std::move(error_listener)); - return chain; } ResumeGuard diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index 7c0d6cd8ee3..ad570202f5b 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -62,7 +62,7 @@ private: void handleLiveConfigUpdate(const InitialGuard & initGuard) override; VisitorMessageSession::UP createSession(Visitor&, VisitorThread&) override; documentapi::Priority::Value toDocumentPriority(uint8_t storagePriority) const override; - std::unique_ptr<StorageLink> createChain() override; + void createChain(IStorageChainBuilder &builder) override; void removeConfigSubscriptions() override; }; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index e962ee4b1b6..aa50391c037 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -12,6 +12,7 @@ #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h> #include <vespa/storage/common/statusmetricconsumer.h> +#include <vespa/storage/common/storage_chain_builder.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/time.h> @@ -111,7 +112,8 @@ StorageNode::StorageNode( _newBucketSpacesConfig(), _component(), _configUri(configUri), - _communicationManager(nullptr) + _communicationManager(nullptr), + _chain_builder(std::make_unique<StorageChainBuilder>()) { } @@ -203,7 +205,9 @@ StorageNode::initialize() _deadLockDetector->setWaitSlack(framework::MilliSecTime( static_cast<uint32_t>(_serverConfig->deadLockDetectorTimeoutSlack * 1000))); - _chain.reset(createChain().release()); + createChain(*_chain_builder); + _chain = std::move(*_chain_builder).build(); + _chain_builder.reset(); assert(_communicationManager != nullptr); _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); @@ -622,4 +626,10 @@ StorageNode::releaseStateManager() { return std::move(_stateManager); } +void +StorageNode::set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder) +{ + _chain_builder = std::move(builder); +} + } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index ea1bf1027d4..91a2bae3190 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -44,6 +44,7 @@ struct DeadLockDetector; struct StorageMetricSet; struct StorageNodeContext; class ApplicationGenerationFetcher; +class IStorageChainBuilder; class StorageComponent; namespace lib { class NodeType; } @@ -164,6 +165,9 @@ protected: std::unique_ptr<StorageComponent> _component; config::ConfigUri _configUri; CommunicationManager* _communicationManager; +private: + std::unique_ptr<IStorageChainBuilder> _chain_builder; +protected: /** * Node subclasses currently need to explicitly acquire ownership of state @@ -177,10 +181,12 @@ protected: void initialize(); virtual void subscribeToConfigs(); virtual void initializeNodeSpecific() = 0; - virtual std::unique_ptr<StorageLink> createChain() = 0; + virtual void createChain(IStorageChainBuilder &builder) = 0; virtual void handleLiveConfigUpdate(const InitialGuard & initGuard); void shutdown(); virtual void removeConfigSubscriptions(); +public: + void set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder); }; } // storage diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 37159ab0011..636f9b1f701 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -5,6 +5,7 @@ #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/stat.h> #include <vespa/storageapi/mbusprot/storageprotocol.h> #include <vespa/storageapi/mbusprot/storagecommand.h> #include <vespa/storageapi/mbusprot/storagereply.h> @@ -567,6 +568,24 @@ TEST_P(StorageProtocolTest, remove_location) { } } +TEST_P(StorageProtocolTest, stat_bucket) { + if (GetParam().getMajor() < 7) { + return; // Only available for protobuf-backed protocol version. + } + auto cmd = std::make_shared<StatBucketCommand>(_bucket, "id.group == 'mygroup'"); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("id.group == 'mygroup'", cmd2->getDocumentSelection()); + EXPECT_EQ(_bucket, cmd2->getBucket()); + + auto reply = std::make_shared<StatBucketReply>(*cmd2, "neat bucket info goes here"); + reply->remapBucketId(_dummy_remap_bucket); + auto reply2 = copyReply(reply); + EXPECT_EQ(reply2->getResults(), "neat bucket info goes here"); + EXPECT_TRUE(reply2->hasBeenRemapped()); + EXPECT_EQ(_dummy_remap_bucket, reply2->getBucketId()); + EXPECT_EQ(_bucket_id, reply2->getOriginalBucketId()); +} + TEST_P(StorageProtocolTest, create_visitor) { std::vector<document::BucketId> buckets; buckets.push_back(document::BucketId(16, 1)); diff --git a/storageapi/src/tests/messageapi/CMakeLists.txt b/storageapi/src/tests/messageapi/CMakeLists.txt index 4833dc45acf..50f0b306191 100644 --- a/storageapi/src/tests/messageapi/CMakeLists.txt +++ b/storageapi/src/tests/messageapi/CMakeLists.txt @@ -1,5 +1,8 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_testmessageapi INTERFACE +vespa_add_library(storageapi_testmessageapi SOURCES + storage_message_address_test.cpp DEPENDS + storageapi + GTest::GTest ) diff --git a/storageapi/src/tests/messageapi/storage_message_address_test.cpp b/storageapi/src/tests/messageapi/storage_message_address_test.cpp new file mode 100644 index 00000000000..c340cba4b28 --- /dev/null +++ b/storageapi/src/tests/messageapi/storage_message_address_test.cpp @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace ::testing; + +namespace storage::api { + +namespace { + +size_t hash_of(vespalib::stringref cluster, const lib::NodeType& type, uint16_t index) { + return StorageMessageAddress(cluster, type, index).internal_storage_hash(); +} + +} + +TEST(StorageMessageAddressTest, storage_hash_covers_all_expected_fields) { + EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("foo", lib::NodeType::STORAGE, 0)); + EXPECT_EQ(hash_of("foo", lib::NodeType::DISTRIBUTOR, 0), + hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); + EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 123), + hash_of("foo", lib::NodeType::STORAGE, 123)); + + // These tests are all true with extremely high probability, though they do + // depend on a hash function that may inherently cause collisions. + EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("bar", lib::NodeType::STORAGE, 0)); + EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); + EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), + hash_of("foo", lib::NodeType::STORAGE, 1)); +} + +} // storage::api diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt index b749844775d..ce5294a7470 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -4,6 +4,7 @@ find_package(Protobuf REQUIRED) PROTOBUF_GENERATE_CPP(storageapi_PROTOBUF_SRCS storageapi_PROTOBUF_HDRS protobuf/common.proto protobuf/feed.proto + protobuf/inspect.proto protobuf/visiting.proto protobuf/maintenance.proto) diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto new file mode 100644 index 00000000000..1bf833e4bf6 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message StatBucketRequest { + Bucket bucket = 1; + bytes document_selection = 2; +} + +message StatBucketResponse { + BucketId remapped_bucket_id = 1; + bytes results = 2; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h index ed7fbc0bdf9..289e5dc355c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h @@ -9,6 +9,7 @@ #endif #include "feed.pb.h" +#include "inspect.pb.h" #include "visiting.pb.h" #include "maintenance.pb.h" diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp index 917b60c50c3..a53bd415c8e 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -7,6 +7,7 @@ #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/util/exceptions.h> @@ -120,6 +121,12 @@ ProtocolSerialization::encode(const api::StorageMessage& msg) const case api::MessageType::VISITOR_DESTROY_REPLY_ID: onEncode(buf, static_cast<const api::DestroyVisitorReply&>(msg)); break; + case api::MessageType::STATBUCKET_ID: + onEncode(buf, static_cast<const api::StatBucketCommand&>(msg)); + break; + case api::MessageType::STATBUCKET_REPLY_ID: + onEncode(buf, static_cast<const api::StatBucketReply&>(msg)); + break; case api::MessageType::REMOVELOCATION_ID: onEncode(buf, static_cast<const api::RemoveLocationCommand&>(msg)); break; @@ -191,6 +198,8 @@ ProtocolSerialization::decodeCommand(mbus::BlobRef data) const cmd = onDecodeCreateVisitorCommand(buf); break; case api::MessageType::VISITOR_DESTROY_ID: cmd = onDecodeDestroyVisitorCommand(buf); break; + case api::MessageType::STATBUCKET_ID: + cmd = onDecodeStatBucketCommand(buf); break; case api::MessageType::REMOVELOCATION_ID: cmd = onDecodeRemoveLocationCommand(buf); break; case api::MessageType::SETBUCKETSTATE_ID: @@ -253,6 +262,8 @@ ProtocolSerialization::decodeReply(mbus::BlobRef data, const api::StorageCommand reply = onDecodeCreateVisitorReply(cmd, buf); break; case api::MessageType::VISITOR_DESTROY_REPLY_ID: reply = onDecodeDestroyVisitorReply(cmd, buf); break; + case api::MessageType::STATBUCKET_REPLY_ID: + reply = onDecodeStatBucketReply(cmd, buf); break; case api::MessageType::REMOVELOCATION_REPLY_ID: reply = onDecodeRemoveLocationReply(cmd, buf); break; case api::MessageType::SETBUCKETSTATE_REPLY_ID: diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h index a57627b9ba9..569ff99c11f 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h @@ -43,6 +43,8 @@ class NotifyBucketChangeCommand; class NotifyBucketChangeReply; class SplitBucketCommand; class SplitBucketReply; +class StatBucketCommand; +class StatBucketReply; class JoinBucketsCommand; class JoinBucketsReply; class SetBucketStateCommand; @@ -111,6 +113,8 @@ protected: virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const = 0; virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const = 0; virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const = 0; + virtual void onEncode(GBBuf&, const api::StatBucketCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::StatBucketReply&) const = 0; virtual SCmd::UP onDecodePutCommand(BBuf&) const = 0; virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const = 0; @@ -148,6 +152,8 @@ protected: virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0; virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeStatBucketCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const = 0; }; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index 8deb689d3a8..13fba8b8508 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -488,6 +488,24 @@ ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf return msg; } +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::StatBucketCommand&) const { + throw vespalib::IllegalStateException("StatBucketCommand not expected for legacy protocol version", VESPA_STRLOC); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeStatBucketCommand(BBuf&) const { + throw vespalib::IllegalStateException("StatBucketCommand not expected for legacy protocol version", VESPA_STRLOC); +} + +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::StatBucketReply&) const { + throw vespalib::IllegalStateException("StatBucketReply not expected for legacy protocol version", VESPA_STRLOC); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeStatBucketReply(const SCmd&, BBuf&) const { + throw vespalib::IllegalStateException("StatBucketReply not expected for legacy protocol version", VESPA_STRLOC); +} + // Utility functions for serialization void diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h index e4ab36dc989..3ae1770f3c2 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h @@ -26,6 +26,8 @@ protected: void onEncode(GBBuf&, const api::DestroyVisitorReply&) const override; void onEncode(GBBuf&, const api::RemoveLocationCommand&) const override; void onEncode(GBBuf&, const api::RemoveLocationReply&) const override; + void onEncode(GBBuf&, const api::StatBucketCommand&) const override; + void onEncode(GBBuf&, const api::StatBucketReply&) const override; // Not supported on 4.2, but implemented here for simplicity. void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; @@ -56,6 +58,8 @@ protected: SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const override; SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; + SCmd::UP onDecodeStatBucketCommand(BBuf&) const override; + SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const override; virtual void onDecodeBucketInfoCommand(BBuf&, api::BucketInfoCommand&) const; virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const = 0; diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 8ea946eede4..166925382bd 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -10,6 +10,7 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/visitor.h> +#include <vespa/storageapi/message/stat.h> namespace storage::mbusprot { @@ -1313,4 +1314,32 @@ api::StorageReply::UP ProtocolSerialization7::onDecodeDestroyVisitorReply(const }); } +// ----------------------------------------------------------------- +// StatBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::StatBucketCommand& msg) const { + encode_bucket_request<protobuf::StatBucketRequest>(buf, msg, [&](auto& req) { + req.set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::StatBucketReply& msg) const { + encode_bucket_response<protobuf::StatBucketResponse>(buf, msg, [&](auto& res) { + res.set_results(msg.getResults().data(), msg.getResults().size()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeStatBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::StatBucketRequest>(buf, [&](auto& req, auto& bucket) { + return std::make_unique<api::StatBucketCommand>(bucket, req.document_selection()); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeStatBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::StatBucketResponse>(buf, [&](auto& res) { + return std::make_unique<api::StatBucketReply>(static_cast<const api::StatBucketCommand&>(cmd), res.results()); + }); +} + } // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h index bb7f0308efa..e1d08691bc1 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h @@ -129,6 +129,12 @@ public: SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; + // StatBucket + void onEncode(GBBuf&, const api::StatBucketCommand&) const override; + void onEncode(GBBuf&, const api::StatBucketReply&) const override; + SCmd::UP onDecodeStatBucketCommand(BBuf&) const override; + SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const override; + private: template <typename ProtobufType, typename Func> std::unique_ptr<api::StorageCommand> decode_request(document::ByteBuffer& in_buf, Func&& f) const; diff --git a/storageapi/src/vespa/storageapi/message/stat.h b/storageapi/src/vespa/storageapi/message/stat.h index 9020d16622a..a7d9a30ca6e 100644 --- a/storageapi/src/vespa/storageapi/message/stat.h +++ b/storageapi/src/vespa/storageapi/message/stat.h @@ -23,7 +23,7 @@ private: public: StatBucketCommand(const document::Bucket &bucket, vespalib::stringref documentSelection); - ~StatBucketCommand(); + ~StatBucketCommand() override; const vespalib::string& getDocumentSelection() const { return _docSelection; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; @@ -33,8 +33,8 @@ public: class StatBucketReply : public BucketReply { vespalib::string _results; public: - StatBucketReply(const StatBucketCommand&, vespalib::stringref results = ""); - const vespalib::string& getResults() { return _results; } + explicit StatBucketReply(const StatBucketCommand&, vespalib::stringref results = ""); + const vespalib::string& getResults() const noexcept { return _results; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGEREPLY(StatBucketReply, onStatBucketReply) }; @@ -51,7 +51,7 @@ public: */ class GetBucketListCommand : public BucketCommand { public: - GetBucketListCommand(const document::Bucket &bucket); + explicit GetBucketListCommand(const document::Bucket &bucket); void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGECOMMAND(GetBucketListCommand, onGetBucketList); }; @@ -78,8 +78,8 @@ private: std::vector<BucketInfo> _buckets; public: - GetBucketListReply(const GetBucketListCommand&); - ~GetBucketListReply(); + explicit GetBucketListReply(const GetBucketListCommand&); + ~GetBucketListReply() override; std::vector<BucketInfo>& getBuckets() { return _buckets; } const std::vector<BucketInfo>& getBuckets() const { return _buckets; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index d1bd24f5087..1f1a2c602de 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <sstream> #include <cassert> #include <atomic> @@ -141,8 +142,8 @@ MessageType::print(std::ostream& out, bool verbose, const std::string& indent) c StorageMessageAddress::StorageMessageAddress(const mbus::Route& route) : _route(route), - _retryEnabled(false), _protocol(DOCUMENT), + _precomputed_storage_hash(0), _cluster(""), _type(nullptr), _index(0xFFFF) @@ -160,17 +161,34 @@ createAddress(vespalib::stringref cluster, const lib::NodeType& type, uint16_t i return os.str(); } +// TODO we ideally want this removed. Currently just in place to support usage as map key when emplacement not available +StorageMessageAddress::StorageMessageAddress() + : _route(), + _protocol(Protocol::STORAGE), + _precomputed_storage_hash(0), + _cluster(), + _type(nullptr), + _index(0) +{} + + StorageMessageAddress::StorageMessageAddress(vespalib::stringref cluster, const lib::NodeType& type, uint16_t index, Protocol protocol) : _route(), - _retryEnabled(false), _protocol(protocol), + _precomputed_storage_hash(0), _cluster(cluster), _type(&type), _index(index) { std::vector<mbus::IHopDirective::SP> directives; - directives.emplace_back(std::make_shared<mbus::VerbatimDirective>(createAddress(cluster, type, index))); + auto address_as_str = createAddress(cluster, type, index); + // We reuse the string representation and pass it to vespalib's hashValue instead of + // explicitly combining a running hash over the individual fields. This is because + // hashValue internally uses xxhash, which offers great dispersion of bits even for + // minimal changes in the input (such as single bit differences in the index). + _precomputed_storage_hash = vespalib::hashValue(address_as_str.data(), address_as_str.size()); + directives.emplace_back(std::make_shared<mbus::VerbatimDirective>(std::move(address_as_str))); _route.addHop(mbus::Hop(std::move(directives), false)); } @@ -207,12 +225,11 @@ bool StorageMessageAddress::operator==(const StorageMessageAddress& other) const { if (_protocol != other._protocol) return false; - if (_retryEnabled != other._retryEnabled) return false; if (_type != other._type) return false; if (_type) { - if (_cluster != other._cluster) return false; if (_index != other._index) return false; if (_type != other._type) return false; + if (_cluster != other._cluster) return false; } return true; } @@ -234,9 +251,6 @@ StorageMessageAddress::print(vespalib::asciistream & out) const } else { out << "Document protocol"; } - if (_retryEnabled) { - out << ", retry enabled"; - } if (!_type) { out << ", " << _route.toString() << ")"; } else { diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index 415bd7717f2..85d4e072171 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -269,14 +269,15 @@ public: private: mbus::Route _route; - bool _retryEnabled; Protocol _protocol; // Used for internal VDS addresses only + size_t _precomputed_storage_hash; vespalib::string _cluster; const lib::NodeType* _type; uint16_t _index; public: + StorageMessageAddress(); // Only to be used when transient default ctor semantics are needed by containers StorageMessageAddress(const mbus::Route& route); StorageMessageAddress(vespalib::stringref clusterName, const lib::NodeType& type, uint16_t index, @@ -284,15 +285,18 @@ public: ~StorageMessageAddress(); void setProtocol(Protocol p) { _protocol = p; } - void enableRetry(bool enable = true) { _retryEnabled = enable; } const mbus::Route& getRoute() const { return _route; } - bool retryEnabled() const { return _retryEnabled; } Protocol getProtocol() const { return _protocol; } uint16_t getIndex() const; const lib::NodeType& getNodeType() const; const vespalib::string& getCluster() const; + // Returns precomputed hash over <cluster, type, index> tuple. Other fields not included. + [[nodiscard]] size_t internal_storage_hash() const noexcept { + return _precomputed_storage_hash; + } + bool operator==(const StorageMessageAddress& other) const; vespalib::string toString() const; friend std::ostream & operator << (std::ostream & os, const StorageMessageAddress & addr); diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp index 4ff3810d85f..bdb53ce6a60 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp @@ -2,6 +2,7 @@ #include "servicelayerprocess.h" #include <vespa/config/helper/configgetter.hpp> +#include <vespa/storage/common/i_storage_chain_builder.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/storageserver/servicelayernode.h> #include <vespa/searchvisitor/searchvisitor.h> @@ -24,6 +25,9 @@ bool configured_to_use_btree_db(const config::ConfigUri& config_uri) { ServiceLayerProcess::ServiceLayerProcess(const config::ConfigUri& configUri) : Process(configUri), + _externalVisitors(), + _node(), + _storage_chain_builder(), _context(std::make_unique<framework::defaultimplementation::RealClock>(), configured_to_use_btree_db(configUri)) { @@ -44,6 +48,9 @@ ServiceLayerProcess::createNode() _externalVisitors["searchvisitor"] = std::make_shared<streaming::SearchVisitorFactory>(_configUri); setupProvider(); _node = std::make_unique<ServiceLayerNode>(_configUri, _context, *this, getProvider(), _externalVisitors); + if (_storage_chain_builder) { + _node->set_storage_chain_builder(std::move(_storage_chain_builder)); + } _node->init(); } @@ -62,4 +69,10 @@ ServiceLayerProcess::getComponentName() const { return "servicelayer"; } +void +ServiceLayerProcess::set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder) +{ + _storage_chain_builder = std::move(builder); +} + } // storage diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h index b24640cbbd7..63776c02a52 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h @@ -27,10 +27,12 @@ namespace storage { namespace spi { struct PersistenceProvider; } class ServiceLayerNode; +class IStorageChainBuilder; class ServiceLayerProcess : public Process { VisitorFactory::Map _externalVisitors; std::unique_ptr<ServiceLayerNode> _node; + std::unique_ptr<IStorageChainBuilder> _storage_chain_builder; protected: ServiceLayerNodeContext _context; @@ -48,6 +50,7 @@ public: StorageNode& getNode() override; StorageNodeContext& getContext() override; std::string getComponentName() const override; + void set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder); }; } // storage diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h index 3bbfb0b23f9..dc7be36c290 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h @@ -73,8 +73,8 @@ public: return vespalib::eval::ConstantValue::UP(); } - std::optional<vespalib::string> getOnnxModelFullPath(const vespalib::string &) const override { - return std::nullopt; + const search::fef::OnnxModel *getOnnxModel(const vespalib::string &) const override { + return nullptr; } bool addField(const vespalib::string & name, bool isAttribute); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 6000026580a..e603a150b34 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -23,6 +23,7 @@ import com.yahoo.document.restapi.RestApiException; import com.yahoo.document.restapi.RestUri; import com.yahoo.document.select.DocumentSelector; import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java index c8ae79deebd..260b57b2112 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java @@ -5,7 +5,7 @@ import java.io.IOException; import java.io.InputStream; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * * @since 5.1.23 */ diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java index 85b714b3b42..9bd2d66fa91 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/XMLFeeder.java @@ -12,7 +12,7 @@ import java.io.InputStream; * access point. * * @author Thomas Gundersen - * @author steinar + * @author Steinar Knutsen */ public class XMLFeeder extends Feeder { public XMLFeeder(DocumentTypeManager docMan, SimpleFeedAccess sender, InputStream stream) { diff --git a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java index 05f7cec9189..de9c88a713d 100644 --- a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java +++ b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java @@ -11,7 +11,7 @@ import java.nio.charset.CharsetEncoder; /** * A buffered writer which accepts byte arrays in addition to character arrays. * - * @author <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class ByteWriter extends AbstractByteWriter { private final OutputStream stream; diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index 51a4fc167c7..144a9a585f6 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -94,8 +94,12 @@ public class SlimeUtils { } public static byte[] toJsonBytes(Slime slime) throws IOException { + return toJsonBytes(slime.get()); + } + + public static byte[] toJsonBytes(Inspector inspector) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new JsonFormat(true).encode(baos, slime); + new JsonFormat(true).encode(baos, inspector); return baos.toByteArray(); } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index c894a798b91..ec003329999 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -26,6 +26,7 @@ vespa_define_module( src/tests/benchmark_timer src/tests/box src/tests/btree + src/tests/child_process src/tests/closure src/tests/component src/tests/compress @@ -46,6 +47,7 @@ vespa_define_module( src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator src/tests/delegatelist + src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache src/tests/dual_merge_director @@ -97,7 +99,6 @@ vespa_define_module( src/tests/sharedptr src/tests/signalhandler src/tests/simple_thread_bundle - src/tests/child_process src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark diff --git a/vespalib/src/tests/detect_type_benchmark/.gitignore b/vespalib/src/tests/detect_type_benchmark/.gitignore new file mode 100644 index 00000000000..3d2fbf713c6 --- /dev/null +++ b/vespalib/src/tests/detect_type_benchmark/.gitignore @@ -0,0 +1 @@ +/vespalib_detect_type_benchmark_app diff --git a/vespalib/src/tests/detect_type_benchmark/CMakeLists.txt b/vespalib/src/tests/detect_type_benchmark/CMakeLists.txt new file mode 100644 index 00000000000..279622dd452 --- /dev/null +++ b/vespalib/src/tests/detect_type_benchmark/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_detect_type_benchmark_app TEST + SOURCES + detect_type_benchmark.cpp + DEPENDS + vespalib + GTest::GTest +) diff --git a/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp new file mode 100644 index 00000000000..6d178093069 --- /dev/null +++ b/vespalib/src/tests/detect_type_benchmark/detect_type_benchmark.cpp @@ -0,0 +1,149 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <typeindex> + +// Typically when you want a flexible way of identifying whether you +// are encountering a specific subclass, you try to dynamic_cast the +// pointer and check for a non-null return value. This is the most +// flexible way since it requires no extra code in the class itself +// and you will also detect any subclasses of the subclass you are +// testing for. Sometimes you only need to identify the exact class +// and speed in doing so is all that matters. This benchmark tries to +// isolate and measure the cost of different strategies. Note that +// dynamic_cast may be more expensive for more complex class +// hierarchies. + +using vespalib::BenchmarkTimer; + +constexpr int A_ID = 1; +constexpr int B_ID = 2; + +constexpr size_t LOOP_CNT = 1000000; + +class BaseClass { +private: + int _static_id; +public: + BaseClass(int id) : _static_id(id) {} + int static_id() const { return _static_id; } + virtual int dynamic_id() const = 0; + virtual ~BaseClass() {} +}; + +struct A : BaseClass { + A() : BaseClass(A_ID) {} + int dynamic_id() const override { return A_ID; } +}; + +struct B : BaseClass { + B() : BaseClass(B_ID) {} + int dynamic_id() const override { return B_ID; } +}; + +using is_A = bool (*)(const BaseClass *); + +//----------------------------------------------------------------------------- + +struct CheckType { + BaseClass *ptr; + is_A pred; + CheckType(BaseClass *ptr_in, is_A pred_in) : ptr(ptr_in), pred(pred_in) {} + void operator()() const { + bool result = pred(ptr); + (void) result; + } +}; + +struct Nop { + void operator()() const {} +}; + +//----------------------------------------------------------------------------- + +A a; +B b; +Nop nop; +double baseline = 0.0; + +//----------------------------------------------------------------------------- + +bool always_true(const BaseClass *) __attribute__((noinline)); +bool always_true(const BaseClass *) { + return true; +} + +bool always_false(const BaseClass *) __attribute__((noinline)); +bool always_false(const BaseClass *) { + return false; +} + +//----------------------------------------------------------------------------- + +bool use_dynamic_cast(const BaseClass *) __attribute__((noinline)); +bool use_dynamic_cast(const BaseClass *ptr) { + return (dynamic_cast<const A*>(ptr)); +} + +bool use_type_index(const BaseClass *) __attribute__((noinline)); +bool use_type_index(const BaseClass *ptr) { + return (std::type_index(typeid(*ptr)) == std::type_index(typeid(A))); +} + +bool use_type_id(const BaseClass *) __attribute__((noinline)); +bool use_type_id(const BaseClass *ptr) { + return (typeid(*ptr) == typeid(A)); +} + +bool use_dynamic_id(const BaseClass *) __attribute__((noinline)); +bool use_dynamic_id(const BaseClass *ptr) { + return (ptr->dynamic_id() == A_ID); +} + +bool use_static_id(const BaseClass *) __attribute__((noinline)); +bool use_static_id(const BaseClass *ptr) { + return (ptr->static_id() == A_ID); +} + +//----------------------------------------------------------------------------- + +double estimate_cost_ns(CheckType check) { + return BenchmarkTimer::benchmark(check, nop, LOOP_CNT, 5.0) * 1000.0 * 1000.0 * 1000.0; +} + +void benchmark(const char *desc, is_A pred) { + EXPECT_TRUE(pred(&a)) << desc; + EXPECT_FALSE(pred(&b)) << desc; + CheckType yes(&a, pred); + CheckType no(&b, pred); + double t1 = estimate_cost_ns(yes); + double t2 = estimate_cost_ns(no); + double my_cost = ((t1 + t2) / 2.0) - baseline; + fprintf(stderr, "%s cost is %5.2f ns (true %5.2f, false %5.2f, baseline %5.2f)\n", + desc, my_cost, t1, t2, baseline); +} + +//----------------------------------------------------------------------------- + +TEST(DetectTypeBenchmark, find_baseline) { + CheckType check_true(&a, always_true); + CheckType check_false(&b, always_false); + double t1 = estimate_cost_ns(check_true); + double t2 = estimate_cost_ns(check_false); + baseline = (t1 + t2) / 2.0; + fprintf(stderr, "baseline cost is %5.2f ns (true %5.2f, false %5.2f)\n", + baseline, t1, t2); +} + +TEST(DetectTypeBenchmark, measure_overhead) { + benchmark("[dynamic_cast]", use_dynamic_cast); + benchmark(" [type_index]", use_type_index); + benchmark(" [typeid]", use_type_id); + benchmark(" [dynamic id]", use_dynamic_id); + benchmark(" [static id]", use_static_id); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 39c0f3f7482..04c4b1e225b 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -155,7 +155,7 @@ DataBuffer::swap(DataBuffer &other) } vespalib::alloc::Alloc -DataBuffer::stealBuffer() +DataBuffer::stealBuffer() && { assert( ! referencesExternalData() ); _externalBuf = nullptr; diff --git a/vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index a520ecd58bd..7c4cd63a7b1 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -606,7 +606,7 @@ public: **/ void swap(DataBuffer &other); - Alloc stealBuffer(); + Alloc stealBuffer() &&; }; } // namespace vespalib diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index f4ad1ab4372..e026559b283 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -105,6 +105,7 @@ ], "methods": [ "public void <init>(java.lang.String, com.yahoo.vespa.curator.Curator)", + "public void <init>(java.lang.String, org.apache.curator.framework.recipes.locks.InterProcessLock)", "public void acquire(java.time.Duration)", "public void close()" ], diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index de542b89b67..7f335467751 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -76,6 +76,11 @@ <artifactId>zookeeper</artifactId> <version>${zookeeper.client.version}</version> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index d97d8f5ed71..920bba22804 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -4,11 +4,11 @@ package com.yahoo.vespa.curator; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.curator.stats.ThreadLockStats; import org.apache.curator.framework.recipes.locks.InterProcessLock; import java.time.Duration; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; /** * A cluster-wide re-entrant mutex which is released on (the last symmetric) close. @@ -20,54 +20,51 @@ import java.util.concurrent.locks.ReentrantLock; */ public class Lock implements Mutex { - private final ReentrantLock lock; private final InterProcessLock mutex; private final String lockPath; public Lock(String lockPath, Curator curator) { + this(lockPath, curator.createMutex(lockPath)); + } + + /** Public for testing only */ + public Lock(String lockPath, InterProcessLock mutex) { this.lockPath = lockPath; - this.lock = new ReentrantLock(true); - mutex = curator.createMutex(lockPath); + this.mutex = mutex; } /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */ public void acquire(Duration timeout) throws UncheckedTimeoutException { + ThreadLockStats threadLockStats = ThreadLockStats.getCurrentThreadLockStats(); + threadLockStats.invokingAcquire(lockPath, timeout); + + final boolean acquired; try { - if ( ! mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS)) - throw new UncheckedTimeoutException("Timed out after waiting " + timeout + - " to acquire lock '" + lockPath + "'"); - if ( ! lock.tryLock()) { // Should be available to only this thread, while holding the above mutex. - release(); - throw new IllegalStateException("InterProcessMutex acquired, but guarded lock held by someone else, for lock '" + lockPath + "'"); - } - } - catch (UncheckedTimeoutException | IllegalStateException e) { - throw e; - } - catch (Exception e) { + acquired = mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (Exception e) { + threadLockStats.acquireFailed(lockPath); throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e); } - } - @Override - public void close() { - try { - lock.unlock(); - } - finally { - release(); + if (!acquired) { + threadLockStats.acquireTimedOut(lockPath); + throw new UncheckedTimeoutException("Timed out after waiting " + timeout + + " to acquire lock '" + lockPath + "'"); } + threadLockStats.lockAcquired(lockPath); } - private void release() { + @Override + public void close() { try { mutex.release(); + ThreadLockStats.getCurrentThreadLockStats().lockReleased(lockPath); } catch (Exception e) { + ThreadLockStats.getCurrentThreadLockStats().lockReleaseFailed(lockPath); throw new RuntimeException("Exception releasing lock '" + lockPath + "'"); } } - } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java new file mode 100644 index 00000000000..3b06377ccf7 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttempt.java @@ -0,0 +1,101 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +/** + * Information about a lock. + * + * <p>Should be mutated by a single thread, except {@link #fillStackTrace()} which can be + * invoked by any threads. Other threads may see an inconsistent state of this instance.</p> + * + * @author hakon + */ +public class LockAttempt { + + private final ThreadLockStats threadLockStats; + private final String lockPath; + private final Instant callAcquireInstant; + private final Duration timeout; + + private volatile Optional<Instant> lockAcquiredInstant = Optional.empty(); + private volatile Optional<Instant> terminalStateInstant = Optional.empty(); + private volatile Optional<String> stackTrace = Optional.empty(); + + public static LockAttempt invokingAcquire(ThreadLockStats threadLockStats, String lockPath, Duration timeout) { + return new LockAttempt(threadLockStats, lockPath, timeout, Instant.now()); + } + + public enum LockState { + ACQUIRING(false), ACQUIRE_FAILED(true), TIMED_OUT(true), ACQUIRED(false), RELEASED(true), + RELEASED_WITH_ERROR(true); + + private final boolean terminal; + + LockState(boolean terminal) { this.terminal = terminal; } + + public boolean isTerminal() { return terminal; } + } + + private volatile LockState lockState = LockState.ACQUIRING; + + private LockAttempt(ThreadLockStats threadLockStats, String lockPath, Duration timeout, Instant callAcquireInstant) { + this.threadLockStats = threadLockStats; + this.lockPath = lockPath; + this.callAcquireInstant = callAcquireInstant; + this.timeout = timeout; + } + + public String getThreadName() { return threadLockStats.getThreadName(); } + public String getLockPath() { return lockPath; } + public Instant getTimeAcquiredWasInvoked() { return callAcquireInstant; } + public Duration getAcquireTimeout() { return timeout; } + public LockState getLockState() { return lockState; } + public Optional<Instant> getTimeLockWasAcquired() { return lockAcquiredInstant; } + public Optional<Instant> getTimeTerminalStateWasReached() { return terminalStateInstant; } + public Optional<String> getStackTrace() { return stackTrace; } + + public Duration getDurationOfAcquire() { + return Duration.between(callAcquireInstant, lockAcquiredInstant.orElseGet(Instant::now)); + } + + public Duration getDurationWithLock() { + return lockAcquiredInstant + .map(start -> Duration.between(start, terminalStateInstant.orElseGet(Instant::now))) + .orElse(Duration.ZERO); + } + + public Duration getDuration() { return Duration.between(callAcquireInstant, terminalStateInstant.orElseGet(Instant::now)); } + + /** Get time from just before trying to acquire lock to the time the terminal state was reached, or ZERO. */ + public Duration getStableTotalDuration() { + return terminalStateInstant.map(instant -> Duration.between(callAcquireInstant, instant)).orElse(Duration.ZERO); + } + + /** Fill in the stack trace starting at the caller's stack frame. */ + public void fillStackTrace() { + // This method is public. If invoked concurrently, the this.stackTrace may be updated twice, + // which is fine. + + this.stackTrace = Optional.of(threadLockStats.getStackTrace()); + } + + void acquireFailed() { setTerminalState(LockState.ACQUIRE_FAILED); } + void timedOut() { setTerminalState(LockState.TIMED_OUT); } + void released() { setTerminalState(LockState.RELEASED); } + void releasedWithError() { setTerminalState(LockState.RELEASED_WITH_ERROR); } + + void lockAcquired() { + lockState = LockState.ACQUIRED; + lockAcquiredInstant = Optional.of(Instant.now()); + } + + void setTerminalState(LockState terminalState) { setTerminalState(terminalState, Instant.now()); } + + void setTerminalState(LockState terminalState, Instant instant) { + lockState = terminalState; + terminalStateInstant = Optional.of(instant); + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttemptSamples.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttemptSamples.java new file mode 100644 index 00000000000..54cb82ebc1e --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockAttemptSamples.java @@ -0,0 +1,106 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; + +/** + * Collection containing "interesting" {@code LockAttempt}s. + * + * @author hakon + */ +// @ThreadSafe +public class LockAttemptSamples { + private final int maxSamples; + + /** Ensure atomic operations on this collection. */ + private final Object monitor = new Object(); + + /** Keep at most one sample for each lock path. */ + private final Map<String, LockAttempt> byLockPath; + + /** + * Priority queue containing all samples. The head of this queue (peek()/poll()) + * returns the LockAttempt with the smallest duration. + */ + private final PriorityQueue<LockAttempt> priorityQueue = + new PriorityQueue<>(Comparator.comparing(LockAttempt::getStableTotalDuration)); + + LockAttemptSamples() { this(10); } + + LockAttemptSamples(int maxSamples) { + this.maxSamples = maxSamples; + this.byLockPath = new HashMap<>(maxSamples); + } + + int size() { return byLockPath.size(); } + + boolean maybeSample(LockAttempt lockAttempt) { + final boolean added; + synchronized (monitor) { + if (shouldAdd(lockAttempt)) { + byLockPath.put(lockAttempt.getLockPath(), lockAttempt); + priorityQueue.add(lockAttempt); + added = true; + } else { + added = false; + } + } + + if (added) { + // Unnecessary to invoke under synchronized, although it means that some samples + // may be without stack trace (just retry if that happens). + lockAttempt.fillStackTrace(); + } + + return added; + } + + private boolean shouldAdd(LockAttempt lockAttempt) { + LockAttempt existingLockAttempt = byLockPath.get(lockAttempt.getLockPath()); + if (existingLockAttempt != null) { + if (hasLongerDurationThan(lockAttempt, existingLockAttempt)) { + byLockPath.remove(existingLockAttempt.getLockPath()); + priorityQueue.remove(existingLockAttempt); + return true; + } + + return false; + } + + if (size() < maxSamples) { + return true; + } + + // peek() and poll() retrieves the smallest element. + existingLockAttempt = priorityQueue.peek(); // cannot be null + if (hasLongerDurationThan(lockAttempt, existingLockAttempt)) { + priorityQueue.poll(); + byLockPath.remove(existingLockAttempt.getLockPath()); + return true; + } + + return false; + } + + List<LockAttempt> asList() { + synchronized (monitor) { + return List.copyOf(byLockPath.values()); + } + } + + void clear() { + synchronized (monitor) { + byLockPath.clear(); + priorityQueue.clear(); + } + } + + private static boolean hasLongerDurationThan(LockAttempt lockAttempt, LockAttempt otherLockAttempt) { + // Use stable total duration to avoid messing up priority queue. + return lockAttempt.getStableTotalDuration().compareTo(otherLockAttempt.getStableTotalDuration()) > 0; + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java new file mode 100644 index 00000000000..561ea9a7ed2 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/LockCounters.java @@ -0,0 +1,66 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A collection of counters for events related to lock acquisition and release. + * + * @author hakon + */ +public class LockCounters { + final AtomicInteger invokeAcquireCount = new AtomicInteger(0); + final AtomicInteger inCriticalRegionCount = new AtomicInteger(0); + final AtomicInteger acquireFailedCount = new AtomicInteger(0); + final AtomicInteger acquireTimedOutCount = new AtomicInteger(0); + final AtomicInteger lockAcquiredCount = new AtomicInteger(0); + final AtomicInteger locksReleasedCount = new AtomicInteger(0); + + final AtomicInteger noLocksErrorCount = new AtomicInteger(0); + final AtomicInteger lockReleaseErrorCount = new AtomicInteger(0); + + public int invokeAcquireCount() { return invokeAcquireCount.get(); } + public int inCriticalRegionCount() { return inCriticalRegionCount.get(); } + public int acquireFailedCount() { return acquireFailedCount.get(); } + public int acquireTimedOutCount() { return acquireTimedOutCount.get(); } + public int lockAcquiredCount() { return lockAcquiredCount.get(); } + public int locksReleasedCount() { return locksReleasedCount.get(); } + public int noLocksErrorCount() { return noLocksErrorCount.get(); } + public int lockReleaseErrorCount() { return lockReleaseErrorCount.get(); } + + @Override + public String toString() { + return "LockCounters{" + + "invokeAcquireCount=" + invokeAcquireCount + + ", inCriticalRegionCount=" + inCriticalRegionCount + + ", acquireFailedCount=" + acquireFailedCount + + ", acquireTimedOutCount=" + acquireTimedOutCount + + ", lockAcquiredCount=" + lockAcquiredCount + + ", locksReleasedCount=" + locksReleasedCount + + ", noLocksErrorCount=" + noLocksErrorCount + + ", locksReleaseErrorCount=" + lockReleaseErrorCount + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LockCounters that = (LockCounters) o; + return invokeAcquireCount.get() == that.invokeAcquireCount.get() && + inCriticalRegionCount.get() == that.inCriticalRegionCount.get() && + acquireFailedCount.get() == that.acquireFailedCount.get() && + acquireTimedOutCount.get() == that.acquireTimedOutCount.get() && + lockAcquiredCount.get() == that.lockAcquiredCount.get() && + locksReleasedCount.get() == that.locksReleasedCount.get() && + noLocksErrorCount.get() == that.noLocksErrorCount.get() && + lockReleaseErrorCount.get() == that.lockReleaseErrorCount.get(); + } + + @Override + public int hashCode() { + return Objects.hash(invokeAcquireCount, inCriticalRegionCount, acquireFailedCount, acquireTimedOutCount, + lockAcquiredCount, locksReleasedCount, noLocksErrorCount, lockReleaseErrorCount); + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java new file mode 100644 index 00000000000..db26523ec37 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/ThreadLockStats.java @@ -0,0 +1,143 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import com.yahoo.vespa.curator.Lock; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.function.Consumer; + +/** + * This class contains process-wide statistics and information related to acquiring and releasing + * {@link Lock}. Instances of this class contain information tied to a specific thread and lock path. + * + * <p>Instances of this class are thread-safe as long as foreign threads (!= this.thread) avoid mutable methods.</p> + * + * @author hakon + */ +public class ThreadLockStats { + + private static final ConcurrentHashMap<Thread, ThreadLockStats> locks = new ConcurrentHashMap<>(); + + private static final LockAttemptSamples COMPLETED_LOCK_ATTEMPT_SAMPLES = new LockAttemptSamples(); + + private static final ConcurrentHashMap<String, LockCounters> countersByLockPath = new ConcurrentHashMap<>(); + + private final Thread thread; + + /** The locks are reentrant so there may be more than 1 lock for this thread. */ + private final ConcurrentLinkedDeque<LockAttempt> lockAttempts = new ConcurrentLinkedDeque<>(); + + public static Map<String, LockCounters> getLockCountersByPath() { return Map.copyOf(countersByLockPath); } + + public static List<ThreadLockStats> getThreadLockStats() { return List.copyOf(locks.values()); } + + public static List<LockAttempt> getLockAttemptSamples() { + return COMPLETED_LOCK_ATTEMPT_SAMPLES.asList(); + } + + /** Returns the per-thread singleton ThreadLockStats. */ + public static ThreadLockStats getCurrentThreadLockStats() { + return locks.computeIfAbsent(Thread.currentThread(), ThreadLockStats::new); + } + + static void clearStaticDataForTesting() { + locks.clear(); + COMPLETED_LOCK_ATTEMPT_SAMPLES.clear(); + countersByLockPath.clear(); + } + + ThreadLockStats(Thread currentThread) { + this.thread = currentThread; + } + + public String getThreadName() { return thread.getName(); } + + public String getStackTrace() { + var stackTrace = new StringBuilder(); + + StackTraceElement[] elements = thread.getStackTrace(); + for (int i = 0; i < elements.length; ++i) { + var element = elements[i]; + stackTrace.append(element.getClassName()) + .append('.') + .append(element.getMethodName()) + .append('(') + .append(element.getFileName()) + .append(':') + .append(element.getLineNumber()) + .append(")\n"); + } + + return stackTrace.toString(); + } + + public List<LockAttempt> getLockAttempts() { return List.copyOf(lockAttempts); } + + /** Mutable method (see class doc) */ + public void invokingAcquire(String lockPath, Duration timeout) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.invokeAcquireCount.incrementAndGet(); + lockCounters.inCriticalRegionCount.incrementAndGet(); + lockAttempts.addLast(LockAttempt.invokingAcquire(this, lockPath, timeout)); + } + + /** Mutable method (see class doc) */ + public void acquireFailed(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.acquireFailedCount.incrementAndGet(); + removeLastLockAttempt(lockCounters, LockAttempt::acquireFailed); + } + + /** Mutable method (see class doc) */ + public void acquireTimedOut(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + + lockCounters.acquireTimedOutCount.incrementAndGet(); + removeLastLockAttempt(lockCounters, LockAttempt::timedOut); + } + + /** Mutable method (see class doc) */ + public void lockAcquired(String lockPath) { + getLockCounters(lockPath).lockAcquiredCount.incrementAndGet(); + LockAttempt lastLockAttempt = lockAttempts.peekLast(); + if (lastLockAttempt == null) { + throw new IllegalStateException("lockAcquired invoked without lockAttempts"); + } + lastLockAttempt.lockAcquired(); + } + + /** Mutable method (see class doc) */ + public void lockReleased(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.locksReleasedCount.incrementAndGet(); + removeLastLockAttempt(lockCounters, LockAttempt::released); + } + + /** Mutable method (see class doc) */ + public void lockReleaseFailed(String lockPath) { + LockCounters lockCounters = getLockCounters(lockPath); + lockCounters.lockReleaseErrorCount.incrementAndGet(); + removeLastLockAttempt(lockCounters, LockAttempt::releasedWithError); + } + + private LockCounters getLockCounters(String lockPath) { + return countersByLockPath.computeIfAbsent(lockPath, __ -> new LockCounters()); + } + + private void removeLastLockAttempt(LockCounters lockCounters, Consumer<LockAttempt> completeLockAttempt) { + lockCounters.inCriticalRegionCount.decrementAndGet(); + + if (lockAttempts.isEmpty()) { + lockCounters.noLocksErrorCount.incrementAndGet(); + return; + } + + LockAttempt lockAttempt = lockAttempts.pollLast(); + completeLockAttempt.accept(lockAttempt); + COMPLETED_LOCK_ATTEMPT_SAMPLES.maybeSample(lockAttempt); + } +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java new file mode 100644 index 00000000000..15a81ffea70 --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/stats/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.curator.stats; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java new file mode 100644 index 00000000000..14dbbad56ba --- /dev/null +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockAttemptSamplesTest.java @@ -0,0 +1,62 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakon + */ +public class LockAttemptSamplesTest { + private final LockAttemptSamples samples = new LockAttemptSamples(2); + private ThreadLockStats threadLockStats; + + @Test + public void test() { + threadLockStats = new ThreadLockStats(Thread.currentThread()); + + assertTrue(maybeSample("1", 10)); + + // new sample has longer duration + assertTrue(maybeSample("1", 11)); + + // new sample has shorter duration + assertFalse(maybeSample("1", 10)); + + // new path, will be added + assertTrue(maybeSample("2", 5)); + + // new path, too low duration be added + assertFalse(maybeSample("3", 4)); + + // new path, expels "2" + assertTrue(maybeSample("4", 6)); + + Map<String, LockAttempt> lockAttempts = samples.asList().stream().collect(Collectors.toMap( + lockAttempt -> lockAttempt.getLockPath(), + lockAttempt -> lockAttempt)); + assertEquals(2, lockAttempts.size()); + + assertTrue(lockAttempts.containsKey("1")); + assertEquals(Duration.ofSeconds(11), lockAttempts.get("1").getStableTotalDuration()); + + assertTrue(lockAttempts.containsKey("4")); + assertEquals(Duration.ofSeconds(6), lockAttempts.get("4").getStableTotalDuration()); + } + + private boolean maybeSample(String lockPath, int secondsDuration) { + LockAttempt lockAttempt = LockAttempt.invokingAcquire(threadLockStats, lockPath, Duration.ofSeconds(1)); + Instant instant = lockAttempt.getTimeAcquiredWasInvoked().plus(Duration.ofSeconds(secondsDuration)); + lockAttempt.setTerminalState(LockAttempt.LockState.RELEASED, instant); + return samples.maybeSample(lockAttempt); + } + +}
\ No newline at end of file diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java new file mode 100644 index 00000000000..92911b0dadf --- /dev/null +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -0,0 +1,140 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; + +import com.yahoo.vespa.curator.Lock; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakon + */ +public class LockTest { + private final InterProcessLock mutex = mock(InterProcessLock.class); + private final String lockPath = "/lock/path"; + private final Duration acquireTimeout = Duration.ofSeconds(10); + private final Lock lock = new Lock(lockPath, mutex); + + @Before + public void setUp() { + ThreadLockStats.clearStaticDataForTesting(); + } + + @Test + public void acquireThrows() throws Exception { + Exception exception = new Exception("example curator exception"); + when(mutex.acquire(anyLong(), any())).thenThrow(exception); + + try { + lock.acquire(acquireTimeout); + fail(); + } catch (Exception e) { + assertSame(e.getCause(), exception); + } + + var expectedCounters = new LockCounters(); + expectedCounters.invokeAcquireCount.set(1); + expectedCounters.acquireFailedCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); + + List<LockAttempt> slowLockAttempts = ThreadLockStats.getLockAttemptSamples(); + assertEquals(1, slowLockAttempts.size()); + LockAttempt slowLockAttempt = slowLockAttempts.get(0); + assertEquals(acquireTimeout, slowLockAttempt.getAcquireTimeout()); + Optional<String> stackTrace = slowLockAttempt.getStackTrace(); + assertTrue(stackTrace.isPresent()); + assertTrue("bad stacktrace: " + stackTrace.get(), stackTrace.get().contains(".Lock.acquire(Lock.java")); + assertEquals(LockAttempt.LockState.ACQUIRE_FAILED, slowLockAttempt.getLockState()); + assertTrue(slowLockAttempt.getTimeTerminalStateWasReached().isPresent()); + + List<ThreadLockStats> threadLockStatsList = ThreadLockStats.getThreadLockStats(); + assertEquals(1, threadLockStatsList.size()); + ThreadLockStats threadLockStats = threadLockStatsList.get(0); + assertEquals(0, threadLockStats.getLockAttempts().size()); + } + + @Test + public void acquireTimesOut() throws Exception { + when(mutex.acquire(anyLong(), any())).thenReturn(false); + + try { + lock.acquire(acquireTimeout); + fail(); + } catch (Exception e) { + assertTrue("unexpected exception: " + e.getMessage(), e.getMessage().contains("Timed out")); + } + + var expectedCounters = new LockCounters(); + expectedCounters.invokeAcquireCount.set(1); + expectedCounters.acquireTimedOutCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); + } + + @Test + public void acquired() throws Exception { + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + lock.acquire(acquireTimeout); + + var expectedCounters = new LockCounters(); + expectedCounters.invokeAcquireCount.set(1); + expectedCounters.lockAcquiredCount.set(1); + expectedCounters.inCriticalRegionCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); + + // reenter + lock.acquire(acquireTimeout); + expectedCounters.invokeAcquireCount.set(2); + expectedCounters.lockAcquiredCount.set(2); + expectedCounters.inCriticalRegionCount.set(2); + + // inner-most closes + lock.close(); + expectedCounters.inCriticalRegionCount.set(1); + expectedCounters.locksReleasedCount.set(1); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); + + // outer-most closes + lock.close(); + expectedCounters.inCriticalRegionCount.set(0); + expectedCounters.locksReleasedCount.set(2); + assertEquals(Map.of(lockPath, expectedCounters), ThreadLockStats.getLockCountersByPath()); + } + + @Test + public void nestedLocks() throws Exception { + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + String lockPath2 = "/lock/path/2"; + Lock lock2 = new Lock(lockPath2, mutex); + + lock.acquire(acquireTimeout); + lock2.acquire(acquireTimeout); + + List<ThreadLockStats> threadLockStats = ThreadLockStats.getThreadLockStats(); + assertEquals(1, threadLockStats.size()); + List<LockAttempt> lockAttempts = threadLockStats.get(0).getLockAttempts(); + assertEquals(2, lockAttempts.size()); + assertEquals(lockPath, lockAttempts.get(0).getLockPath()); + assertEquals(LockAttempt.LockState.ACQUIRED, lockAttempts.get(0).getLockState()); + assertEquals(lockPath2, lockAttempts.get(1).getLockPath()); + assertEquals(LockAttempt.LockState.ACQUIRED, lockAttempts.get(1).getLockState()); + + lock.close(); + lock.close(); + } +} |