diff options
270 files changed, 5997 insertions, 2579 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java index 6bdc7a949e7..801213dcf40 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceId.java @@ -12,6 +12,10 @@ import java.util.Objects; public class ApplicationInstanceId { public static final ApplicationInstanceId CONFIG_SERVER = new ApplicationInstanceId("zone-config-servers"); public static final ApplicationInstanceId CONTROLLER = new ApplicationInstanceId("controller"); + // Unfortunately, for config server host the ApplicationInstanceId is: configserver-host:prod:cd-us-central-1:default + public boolean isConfigServerHost() { return id.startsWith("configserver-host:"); } + public static final ApplicationInstanceId CONTROLLER_HOST = new ApplicationInstanceId("controller-host:prod:default:default"); + public boolean isTenantHost() { return id.startsWith("tenant-host:"); } private final String id; diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java index e761e14caa4..f0b2c46d460 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstanceReference.java @@ -10,7 +10,7 @@ import java.util.Objects; * @author bjorncs */ // TODO: Remove this and use ApplicationId instead (if you need it for the JSON stuff move it to that layer and don't let it leak) -public class ApplicationInstanceReference { +public class ApplicationInstanceReference implements Comparable<ApplicationInstanceReference> { private final TenantId tenantId; private final ApplicationInstanceId applicationInstanceId; @@ -43,6 +43,11 @@ public class ApplicationInstanceReference { } @Override + public int compareTo(ApplicationInstanceReference o) { + return this.asString().compareTo(o.asString()); + } + + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java index 96be7090114..d7995b41616 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ClusterId.java @@ -12,6 +12,9 @@ public class ClusterId { public static final ClusterId CONFIG_SERVER = new ClusterId("zone-config-servers"); public static final ClusterId CONTROLLER = new ClusterId("controller"); + public static final ClusterId CONFIG_SERVER_HOST = new ClusterId("configserver-host"); + public static final ClusterId CONTROLLER_HOST = new ClusterId("controller-host"); + public static final ClusterId TENANT_HOST = new ClusterId("tenant-host"); private final String id; diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java index 1faefcb7c61..43f161cfec9 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Predicate; /** * Represents a collection of service instances that together make up a service with a single cluster id. @@ -51,24 +52,53 @@ public class ServiceCluster { return applicationInstance.get(); } - public boolean isConfigServerClusterLike() { - // config server - if (Objects.equals(applicationInstance.map(ApplicationInstance::tenantId), Optional.of(TenantId.HOSTED_VESPA)) && - Objects.equals(applicationInstance.map(ApplicationInstance::applicationInstanceId), Optional.of(ApplicationInstanceId.CONFIG_SERVER)) && - Objects.equals(clusterId, ClusterId.CONFIG_SERVER) && - Objects.equals(serviceType, ServiceType.CONFIG_SERVER)) { - return true; - } + public boolean isConfigServerLike() { + return isConfigServer() || isController(); + } - // controller - if (Objects.equals(applicationInstance.map(ApplicationInstance::tenantId), Optional.of(TenantId.HOSTED_VESPA)) && - Objects.equals(applicationInstance.map(ApplicationInstance::applicationInstanceId), Optional.of(ApplicationInstanceId.CONTROLLER)) && + public boolean isController() { + return isHostedVespaApplicationWithId(ApplicationInstanceId.CONTROLLER) && Objects.equals(clusterId, ClusterId.CONTROLLER) && - Objects.equals(serviceType, ServiceType.CONTROLLER)) { - return true; - } + Objects.equals(serviceType, ServiceType.CONTROLLER); + } + + /** Is a config server (and not controller!) */ + public boolean isConfigServer() { + return isHostedVespaApplicationWithId(ApplicationInstanceId.CONFIG_SERVER) && + Objects.equals(clusterId, ClusterId.CONFIG_SERVER) && + Objects.equals(serviceType, ServiceType.CONFIG_SERVER); + } + + public boolean isConfigServerHost() { + return isHostedVespaApplicationWithPredicate(ApplicationInstanceId::isConfigServerHost) && + Objects.equals(clusterId, ClusterId.CONFIG_SERVER_HOST) && + Objects.equals(serviceType, ServiceType.HOST_ADMIN); + } + + public boolean isControllerHost() { + return isHostedVespaApplicationWithId(ApplicationInstanceId.CONTROLLER_HOST) && + Objects.equals(clusterId, ClusterId.CONTROLLER_HOST) && + Objects.equals(serviceType, ServiceType.HOST_ADMIN); + } + + public boolean isTenantHost() { + return isHostedVespaApplicationWithPredicate(ApplicationInstanceId::isTenantHost) && + Objects.equals(clusterId, ClusterId.TENANT_HOST) && + Objects.equals(serviceType, ServiceType.HOST_ADMIN); + } + + private boolean isHostedVespaApplicationWithId(ApplicationInstanceId id) { + return isHostedVespaTenant() && + applicationInstance.map(app -> Objects.equals(app.applicationInstanceId(), id)).orElse(false); + } + + private boolean isHostedVespaApplicationWithPredicate(Predicate<ApplicationInstanceId> predicate) { + return isHostedVespaTenant() && + applicationInstance.map(app -> predicate.test(app.applicationInstanceId())).orElse(false); + } - return false; + private boolean isHostedVespaTenant() { + return applicationInstance.map(a -> Objects.equals(a.tenantId(), TenantId.HOSTED_VESPA)).orElse(false); } @Override diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java index b4fce878b0d..d75d3abd5da 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceInstance.java @@ -66,6 +66,27 @@ public class ServiceInstance { '}'; } + /** + * Get a name that can be used in e.g. config server logs that makes it easy to understand which + * service instance this is. + */ + public String descriptiveName() { + if (getServiceCluster().isController() || getServiceCluster().isConfigServer()) { + return getHostnamePrefix(); + } else if (getServiceCluster().isControllerHost() || getServiceCluster().isConfigServerHost()) { + return "host-admin on " + getHostnamePrefix(); + } else if (getServiceCluster().isTenantHost()) { + return "host-admin on " + hostName.s(); + } else { + return configId.s(); + } + } + + private String getHostnamePrefix() { + int dotIndex = hostName.s().indexOf('.'); + return dotIndex == -1 ? hostName().s() : hostName.s().substring(0, dotIndex); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 7d0228721aa..756961933db 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -89,6 +89,9 @@ public interface ModelContext { boolean useThreePhaseUpdates(); + // TODO Remove on 7.XXX when this is default on. + boolean useDirectStorageApiRpc(); + default String proxyProtocol() { return "https+proxy-protocol"; } // TODO bjorncs: Remove after end of May default Optional<AthenzDomain> athenzDomain() { return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index fc799449379..22bdf31350a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -37,6 +37,7 @@ public class TestProperties implements ModelContext.Properties { private boolean useDedicatedNodeForLogserver = false; private boolean useContentNodeBtreeDb = false; private boolean useThreePhaseUpdates = false; + private boolean useDirectStorageApiRpc = false; private double defaultTermwiseLimit = 1.0; private double threadPoolSizeFactor = 0.0; private double queueSizeFactor = 0.0; @@ -73,6 +74,7 @@ public class TestProperties implements ModelContext.Properties { } @Override public boolean useContentNodeBtreeDb() { return useContentNodeBtreeDb; } @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } + @Override public boolean useDirectStorageApiRpc() { return useDirectStorageApiRpc; } @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } @Override public Optional<ApplicationRoles> applicationRoles() { return Optional.ofNullable(applicationRoles); } @Override public String responseSequencerType() { return responseSequencerType; } @@ -113,6 +115,11 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setUseDirectStorageApiRpc(boolean useDirectStorageApiRpc) { + this.useDirectStorageApiRpc = useDirectStorageApiRpc; + return this; + } + public TestProperties setThreadPoolSizeFactor(double threadPoolSizeFactor) { this.threadPoolSizeFactor = threadPoolSizeFactor; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 1b5be1c2f97..3f432620b90 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -31,7 +31,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import static com.yahoo.vespa.model.admin.monitoring.MetricSet.emptyMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty; /** * This is the admin pseudo-plugin of the Vespa model, responsible for @@ -49,7 +49,7 @@ public class Admin extends AbstractConfigProducer implements Serializable { private final Metrics metrics; private MetricsProxyContainerCluster metricsProxyCluster; - private MetricSet additionalDefaultMetrics = emptyMetricSet(); + private MetricSet additionalDefaultMetrics = empty(); private final List<Slobrok> slobroks = new ArrayList<>(); private Configserver defaultConfigserver; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java index 3f9edae10c0..2c039118cb9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java @@ -12,8 +12,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; - /** * Helper class to generate config for metrics consumers. * @@ -22,14 +20,16 @@ import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_ class ConsumersConfigGenerator { /** - * @param userConsumers The consumers set up by the user in services.xml - * @return A list of consumer builders (a mapping from consumer to its metrics) + * @param userConsumers the consumers set up by the user in services.xml + * @return a list of consumer builders (a mapping from consumer to its metrics) */ static List<Consumer.Builder> generateConsumers(MetricsConsumer defaultConsumer, Map<String, MetricsConsumer> userConsumers) { - // Normally, the user given consumers should not contain VESPA_CONSUMER_ID, but it's allowed for some internally used applications. + // Normally, the user given consumers should not contain VESPA_CONSUMER_ID, + // but it's allowed for some internally used applications. var allConsumers = new LinkedHashMap<>(userConsumers); - allConsumers.put(VESPA_CONSUMER_ID, combineConsumers(defaultConsumer, allConsumers.get(VESPA_CONSUMER_ID))); + allConsumers.put(MetricsConsumer.vespa.id(), + combineConsumers(defaultConsumer, allConsumers.get(MetricsConsumer.vespa.id()))); return allConsumers.values().stream() .map(ConsumersConfigGenerator::toConsumerBuilder) @@ -45,18 +45,18 @@ class ConsumersConfigGenerator { */ private static MetricsConsumer combineConsumers(MetricsConsumer original, MetricsConsumer overriding) { if (overriding == null) return original; - return addMetrics(original, overriding.getMetrics()); + return addMetrics(original, overriding.metrics()); } static MetricsConsumer addMetrics(MetricsConsumer original, Map<String, Metric> metrics) { if (metrics == null) return original; - Map<String, Metric> combinedMetrics = new LinkedHashMap<>(original.getMetrics()); + Map<String, Metric> combinedMetrics = new LinkedHashMap<>(original.metrics()); metrics.forEach((name, newMetric) -> - combinedMetrics.put(name, combineMetrics(original.getMetrics().get(name), newMetric))); + combinedMetrics.put(name, combineMetrics(original.metrics().get(name), newMetric))); - return new MetricsConsumer(original.getId(), - new MetricSet(original.getMetricSet().getId(), combinedMetrics.values())); + return new MetricsConsumer(original.id(), + new MetricSet(original.metricSet().getId(), combinedMetrics.values())); } private static Metric combineMetrics(Metric original, Metric newMetric) { @@ -64,8 +64,8 @@ class ConsumersConfigGenerator { } static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) { - Consumer.Builder builder = new Consumer.Builder().name(consumer.getId()); - consumer.getMetrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); + Consumer.Builder builder = new Consumer.Builder().name(consumer.id()); + consumer.metrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index b5936887b50..fbf6dcfd5eb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -56,9 +56,7 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.SYSTEM; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.getDefaultPublicConsumer; -import static com.yahoo.vespa.model.admin.monitoring.MetricSet.emptyMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.getVespaMetricsConsumer; +import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty; /** * Container cluster for metrics proxy containers. @@ -161,10 +159,10 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC @Override public void getConfig(ConsumersConfig.Builder builder) { - var amendedVespaConsumer = addMetrics(getVespaMetricsConsumer(), getAdditionalDefaultMetrics().getMetrics()); + var amendedVespaConsumer = addMetrics(MetricsConsumer.vespa, getAdditionalDefaultMetrics().getMetrics()); builder.consumer.addAll(generateConsumers(amendedVespaConsumer, getUserMetricsConsumers())); - builder.consumer.add(toConsumerBuilder(getDefaultPublicConsumer())); + builder.consumer.add(toConsumerBuilder(MetricsConsumer.defaultConsumer)); } @Override @@ -210,7 +208,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private MetricSet getAdditionalDefaultMetrics() { return getAdmin() .map(Admin::getAdditionalDefaultMetrics) - .orElse(emptyMetricSet()); + .orElse(empty()); } // Returns the metrics consumers from services.xml diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java new file mode 100644 index 00000000000..f6f51ed91fb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.monitoring; + +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Metrics used for autoscaling + * + * @author bratseth + */ +public class AutoscalingMetrics { + + public static final MetricSet autoscalingMetricSet = create(); + + private static MetricSet create() { + return new MetricSet("autoscaling", + metrics("cpu.util", "mem_total.util", "disk.util")); + } + + private static Set<Metric> metrics(String ... names) { + return Arrays.stream(names).map(Metric::new).collect(Collectors.toSet()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java index 5351c3fb3a7..66a278b3fb6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java @@ -8,6 +8,7 @@ import java.util.Optional; * @author gjoranv */ public class CloudWatch { + private final String region; private final String namespace; private final MetricsConsumer consumer; @@ -23,7 +24,7 @@ public class CloudWatch { public String region() { return region; } public String namespace() { return namespace; } - public String consumer() { return consumer.getId(); } + public String consumer() { return consumer.id(); } public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); } public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index c80cebe3d5b..adc0d0b1db8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -9,7 +9,6 @@ import java.util.LinkedHashSet; import java.util.Set; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static java.util.Collections.emptyList; import static java.util.Collections.singleton; /** @@ -18,14 +17,14 @@ import static java.util.Collections.singleton; * * @author gjoranv */ -public class DefaultPublicMetrics { +public class DefaultMetrics { - public static final String DEFAULT_METRIC_SET_ID = "default"; + public static final String defaultMetricSetId = "default"; - public static MetricSet defaultPublicMetricSet = createMetricSet(); + public static MetricSet defaultMetricSet = createMetricSet(); private static MetricSet createMetricSet() { - return new MetricSet(DEFAULT_METRIC_SET_ID, + return new MetricSet(defaultMetricSetId, getAllMetrics(), singleton(defaultVespaMetricSet)); } @@ -95,6 +94,6 @@ public class DefaultPublicMetrics { return metrics; } - private DefaultPublicMetrics() { } + private DefaultMetrics() { } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicConsumer.java deleted file mode 100644 index 68b1fc3c983..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicConsumer.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metricsproxy.http.ValuesFetcher; -import com.google.common.collect.ImmutableList; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static java.util.Collections.emptyList; - -/** - * @author gjoranv - */ -public class DefaultPublicConsumer { - - public static final String DEFAULT_PUBLIC_CONSUMER_ID = ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID.id; - - private static final MetricSet publicConsumerMetrics = new MetricSet("public-consumer-metrics", - emptyList(), - ImmutableList.of(defaultPublicMetricSet, - systemMetricSet)); - - public static MetricsConsumer getDefaultPublicConsumer() { - return new MetricsConsumer(DEFAULT_PUBLIC_CONSUMER_ID, publicConsumerMetrics); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java index d879a6f445d..30797f27789 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java @@ -24,21 +24,19 @@ public class MetricSet { private final Map<String, Metric> metrics; private final Set<MetricSet> children; + public MetricSet(String id, Collection<Metric> metrics) { + this(id, metrics, Collections.emptySet()); + } public MetricSet(String id, Collection<Metric> metrics, Collection<MetricSet> children) { - Objects.requireNonNull(id, "Id cannot be null or empty."); + this.id = Objects.requireNonNull(id, "Id cannot be null or empty."); - this.id = id; this.metrics = toMapByName(metrics); this.children = new LinkedHashSet<>(children); } - public MetricSet(String id, Collection<Metric> metrics) { - this(id, metrics, Collections.emptySet()); - } - - public static MetricSet emptyMetricSet() { - return new MetricSet("empty", Collections.emptySet()); + public static MetricSet empty() { + return new MetricSet("empty", Set.of()); } public final String getId() { return id; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index a8fbcf50b02..698b01c306a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -1,26 +1,44 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import javax.annotation.concurrent.Immutable; +import ai.vespa.metricsproxy.core.VespaMetrics; +import ai.vespa.metricsproxy.http.ValuesFetcher; + import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; +import static com.yahoo.vespa.model.admin.monitoring.AutoscalingMetrics.autoscalingMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; import static java.util.Collections.unmodifiableList; /** - * Represents an arbitrary metric consumer + * A metric consumer is a set of metrics given an id that can be requested at runtime. * - * @author trygve + * @author Trygve Berdal * @author gjoranv */ -@Immutable +// TODO: This construct seems redundant when we have metrics sets public class MetricsConsumer { + // Pre-defined consumers + public static final MetricsConsumer vespa = + consumer(VespaMetrics.vespaMetricsConsumerId.id, vespaMetricSet, systemMetricSet, networkMetricSet); + public static final MetricsConsumer defaultConsumer = + consumer(ValuesFetcher.defaultMetricsConsumerId.id, defaultMetricSet, systemMetricSet); + // Referenced from com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsFetcher + public static final MetricsConsumer autoscaling = + consumer("autoscaling", autoscalingMetricSet); + private final String id; private final MetricSet metricSet; + // TODO: This shouldn't be here private final List<CloudWatch> cloudWatches = new ArrayList<>(); /** @@ -32,16 +50,16 @@ public class MetricsConsumer { this.metricSet = Objects.requireNonNull(metricSet, "A consumer must have a non-null metric set."); } - public String getId() { + public String id() { return id; } - public MetricSet getMetricSet() { return metricSet; } + public MetricSet metricSet() { return metricSet; } /** - * @return Map of metric with metric name as key + * @return map of metric with metric name as key */ - public Map<String, Metric> getMetrics() { + public Map<String, Metric> metrics() { return metricSet.getMetrics(); } @@ -53,4 +71,8 @@ public class MetricsConsumer { return unmodifiableList(cloudWatches); } + private static MetricsConsumer consumer(String id, MetricSet ... metricSets) { + return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), Arrays.asList(metricSets))); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricsConsumer.java deleted file mode 100644 index 9f3bfdc8ae8..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricsConsumer.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import ai.vespa.metricsproxy.core.VespaMetrics; -import com.google.common.collect.ImmutableList; - -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static java.util.Collections.emptyList; - -/** - * This class sets up the 'Vespa' metrics consumer, which is mainly used for Yamas in hosted Vespa. - * - * @author trygve - * @author gjoranv - */ -public class VespaMetricsConsumer { - - public static final String VESPA_CONSUMER_ID = VespaMetrics.VESPA_CONSUMER_ID.id; - - private static final MetricSet vespaConsumerMetrics = new MetricSet("vespa-consumer-metrics", - emptyList(), - ImmutableList.of(vespaMetricSet, - systemMetricSet, - networkMetricSet)); - - public static MetricsConsumer getVespaMetricsConsumer() { - return new MetricsConsumer(VESPA_CONSUMER_ID, vespaConsumerMetrics); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java index 1f81f16a80b..6f9e9a08aa6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java @@ -17,7 +17,7 @@ public class Metrics { private final Map<String, MetricsConsumer> consumers = new LinkedHashMap<>(); public void addConsumer(MetricsConsumer consumer) { - consumers.put(consumer.getId(), consumer); + consumers.put(consumer.id(), consumer); } public Map<String, MetricsConsumer> getConsumers() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java index 694108d4bb1..dd514f1e245 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java @@ -7,7 +7,8 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.AutoscalingMetrics.autoscalingMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; @@ -20,14 +21,17 @@ import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricS */ public class PredefinedMetricSets { - public static final Map<String, MetricSet> predefinedMetricSets = toMapById( - defaultPublicMetricSet, + private static final Map<String, MetricSet> sets = toMapById( + defaultMetricSet, defaultVespaMetricSet, vespaMetricSet, systemMetricSet, - networkMetricSet + networkMetricSet, + autoscalingMetricSet ); + public static Map<String, MetricSet> get() { return sets; } + private static Map<String, MetricSet> toMapById(MetricSet... metricSets) { Map<String, MetricSet> availableMetricSets = new LinkedHashMap<>(); for (MetricSet metricSet : metricSets) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index b686288868f..919830d912f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -14,8 +14,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; @@ -85,11 +83,11 @@ public class MetricsBuilder { } private void throwIfIllegalConsumerId(Metrics metrics, String consumerId) { - if (consumerId.equalsIgnoreCase(VESPA_CONSUMER_ID) && applicationType != ApplicationType.HOSTED_INFRASTRUCTURE) + if (consumerId.equalsIgnoreCase(MetricsConsumer.vespa.id()) && applicationType != ApplicationType.HOSTED_INFRASTRUCTURE) throw new IllegalArgumentException("'Vespa' is not allowed as metrics consumer id (case is ignored.)"); - if (consumerId.equalsIgnoreCase(DEFAULT_PUBLIC_CONSUMER_ID)) - throw new IllegalArgumentException("'" + DEFAULT_PUBLIC_CONSUMER_ID + "' is not allowed as metrics consumer id (case is ignored.)"); + if (consumerId.equalsIgnoreCase(MetricsConsumer.defaultConsumer.id())) + throw new IllegalArgumentException("'" + MetricsConsumer.defaultConsumer.id() + "' is not allowed as metrics consumer id (case is ignored.)"); if (metrics.hasConsumerIgnoreCase(consumerId)) throw new IllegalArgumentException("'" + consumerId + "' is used as id for two metrics consumers (case is ignored.)"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java index 462ac39fa84..025b7875677 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java @@ -31,7 +31,7 @@ public class CloudWatchValidator extends Validator { } private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) { - return offendingConsumers.stream().map(MetricsConsumer::getId).collect(toList()); + return offendingConsumers.stream().map(MetricsConsumer::id).collect(toList()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index d2f06da992c..b643f771d73 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.model.admin.ModelConfigProvider; import com.yahoo.vespa.model.admin.monitoring.DefaultMonitoring; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; +import com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets; import com.yahoo.vespa.model.admin.monitoring.builder.xml.MetricsBuilder; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import org.w3c.dom.Element; @@ -25,8 +26,6 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.List; -import static com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets.predefinedMetricSets; - /** * A base class for admin model builders, to support common functionality across versions. * @@ -69,8 +68,8 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu @Override protected Admin doBuild(DeployState deployState, AbstractConfigProducer parent, Element adminElement) { Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring"), deployState.isHosted()); - Metrics metrics = new MetricsBuilder(applicationType, predefinedMetricSets) - .buildMetrics(XML.getChild(adminElement, "metrics")); + Metrics metrics = new MetricsBuilder(applicationType, PredefinedMetricSets.get()) + .buildMetrics(XML.getChild(adminElement, "metrics")); FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent, deployState.isHosted()); Admin admin = new Admin(parent, monitoring, metrics, multitenant, fileDistributionConfigProducer, deployState.isHosted()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 9012cc2aba0..feaa6eb5940 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -3,17 +3,16 @@ package com.yahoo.vespa.model.clients; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.ThreadPoolExecutorComponent; +import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; import java.util.Collection; import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; /** * @author Einar M R Rosenvinge @@ -24,51 +23,31 @@ public class ContainerDocumentApi { private static final int FALLBACK_MAX_POOL_SIZE = 0; // Use fallback based on actual logical core count on host private static final int FALLBACK_CORE_POOL_SIZE = 0; // Use fallback based on actual logical core count on host - private final ContainerCluster<?> cluster; - private final Options options; - private final Handler<AbstractConfigProducer<?>> feedHandler; - private final Handler<AbstractConfigProducer<?>> restApiHandler; - public ContainerDocumentApi(ContainerCluster<?> cluster, Options options) { - this.cluster = cluster; - this.options = options; - this.restApiHandler = addRestApiHandler(cluster, options); - this.feedHandler = addFeedHandler(cluster, options); - } - - - public void addNodesDependentThreadpoolConfiguration() { - if (cluster.getContainers().isEmpty()) throw new IllegalStateException("Cluster is empty"); - ThreadPoolExecutorComponent feedHandlerExecutor = newExecutorComponent("feedapi-handler", cluster, options); - feedHandler.inject(feedHandlerExecutor); - feedHandler.addComponent(feedHandlerExecutor); - ThreadPoolExecutorComponent restApiHandlerExecutor = newExecutorComponent("restapi-handler", cluster, options); - restApiHandler.inject(restApiHandlerExecutor); - restApiHandler.addComponent(restApiHandlerExecutor); + addRestApiHandler(cluster, options); + addFeedHandler(cluster, options); } - private static Handler<AbstractConfigProducer<?>> addFeedHandler(ContainerCluster<?> cluster, Options options) { + private static void addFeedHandler(ContainerCluster<?> cluster, Options options) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; var handler = newVespaClientHandler( "com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, options); cluster.addComponent(handler); - return handler; + var executor = new Threadpool( + "feedapi-handler", cluster, options.feedApiThreadpoolOptions, options.feedThreadPoolSizeFactor); + handler.inject(executor); + handler.addComponent(executor); } - private static Handler<AbstractConfigProducer<?>> addRestApiHandler(ContainerCluster<?> cluster, Options options) { + private static void addRestApiHandler(ContainerCluster<?> cluster, Options options) { var handler = newVespaClientHandler( "com.yahoo.document.restapi.resource.RestApi", "/document/v1/*", options); cluster.addComponent(handler); - return handler; - } - - private static ThreadPoolExecutorComponent newExecutorComponent(String name, ContainerCluster<?> cluster, Options options) { - return new ThreadPoolExecutorComponent.Builder(name) - .maxPoolSize(maxPoolSize(cluster, options)) - .corePoolSize(corePoolSize(cluster, options)) - .queueSize(500) - .build(); + var executor = new Threadpool( + "restapi-handler", cluster, options.restApiThreadpoolOptions, options.feedThreadPoolSizeFactor); + handler.inject(executor); + handler.addComponent(executor); } private static Handler<AbstractConfigProducer<?>> newVespaClientHandler( @@ -92,37 +71,60 @@ public class ContainerDocumentApi { return handler; } - private static int maxPoolSize(ContainerCluster<?> cluster, Options options) { - double vcpu = vcpu(cluster); - if (vcpu == 0) return FALLBACK_MAX_POOL_SIZE; - return Math.max(2, (int)Math.ceil(vcpu * options.feedThreadPoolSizeFactor)); - } + public static final class Options { + private final Collection<String> bindings; + private final ContainerThreadpool.UserOptions restApiThreadpoolOptions; + private final ContainerThreadpool.UserOptions feedApiThreadpoolOptions; + private final double feedThreadPoolSizeFactor; - private static int corePoolSize(ContainerCluster<?> cluster, Options options) { - double vcpu = vcpu(cluster); - if (vcpu == 0) return FALLBACK_CORE_POOL_SIZE; - return Math.max(1, (int)Math.ceil(vcpu * options.feedThreadPoolSizeFactor * 0.5)); + public Options(Collection<String> bindings, + ContainerThreadpool.UserOptions restApiThreadpoolOptions, + ContainerThreadpool.UserOptions feedApiThreadpoolOptions, + double feedThreadPoolSizeFactor) { + this.bindings = Collections.unmodifiableCollection(bindings); + this.restApiThreadpoolOptions = restApiThreadpoolOptions; + this.feedApiThreadpoolOptions = feedApiThreadpoolOptions; + this.feedThreadPoolSizeFactor = feedThreadPoolSizeFactor; + } } - private static double vcpu(ContainerCluster<?> cluster) { - List<Double> vcpus = cluster.getContainers().stream() - .filter(c -> c.getHostResource() != null && c.getHostResource().realResources() != null) - .map(c -> c.getHostResource().realResources().vcpu()) - .distinct() - .collect(Collectors.toList()); - // We can only use host resource for calculation if all container nodes in the cluster are homogeneous (in terms of vcpu) - if (vcpus.size() != 1 || vcpus.get(0) == 0) return 0; - return vcpus.get(0); - } + private static class Threadpool extends ContainerThreadpool { - public static final class Options { - private final Collection<String> bindings; + private final ContainerCluster<?> cluster; private final double feedThreadPoolSizeFactor; - public Options(Collection<String> bindings, double feedThreadPoolSizeFactor) { - this.bindings = Collections.unmodifiableCollection(bindings); + Threadpool(String name, + ContainerCluster<?> cluster, + ContainerThreadpool.UserOptions threadpoolOptions, + double feedThreadPoolSizeFactor ) { + super(name, threadpoolOptions); + this.cluster = cluster; this.feedThreadPoolSizeFactor = feedThreadPoolSizeFactor; } + + @Override + public void getConfig(ContainerThreadpoolConfig.Builder builder) { + super.getConfig(builder); + + // User options overrides below configuration + if (hasUserOptions()) return; + + builder.maxThreads(maxPoolSize()); + builder.minThreads(minPoolSize()); + builder.queueSize(500); + } + + private int maxPoolSize() { + double vcpu = vcpu(cluster); + if (vcpu == 0) return FALLBACK_MAX_POOL_SIZE; + return Math.max(2, (int)Math.ceil(vcpu * feedThreadPoolSizeFactor)); + } + + private int minPoolSize() { + double vcpu = vcpu(cluster); + if (vcpu == 0) return FALLBACK_CORE_POOL_SIZE; + return Math.max(1, (int)Math.ceil(vcpu * feedThreadPoolSizeFactor * 0.5)); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 5207a0163cb..232552ea4ce 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.model.container; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.NodeResources; import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.search.config.QrStartConfig; @@ -15,10 +15,7 @@ import com.yahoo.vespa.model.container.component.Component; * * @author gjoranv */ -public final class ApplicationContainer extends Container implements - QrStartConfig.Producer, - ThreadpoolConfig.Producer -{ +public final class ApplicationContainer extends Container implements QrStartConfig.Producer { private static final String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; @@ -44,9 +41,9 @@ public final class ApplicationContainer extends Container implements @Override public void getConfig(QrStartConfig.Builder builder) { if (getHostResource() != null) { - if ( ! getHostResource().realResources().isUnspecified()) { - NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().realResources()); - flavorTuning.getConfig(builder); + NodeResources nodeResources = getHostResource().realResources(); + if ( ! nodeResources.isUnspecified()) { + builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(nodeResources.vcpu()))); } } } @@ -75,17 +72,4 @@ public final class ApplicationContainer extends Container implements private boolean hasDocproc() { return (parent instanceof ContainerCluster) && (((ContainerCluster)parent).getDocproc() != null); } - - @Override - public void getConfig(ThreadpoolConfig.Builder builder) { - if (! (parent instanceof ContainerCluster)) return; - if ((getHostResource() == null) || getHostResource().realResources().isUnspecified()) return; - ContainerCluster containerCluster = (ContainerCluster) parent; - if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return; - - NodeResourcesTuning resourcesTuning = new NodeResourcesTuning(getHostResource().realResources()) - .setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor()) - .setQueueSizeFactor(containerCluster.getQueueSizeFactor()); - resourcesTuning.getConfig(builder); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6d8f3056cef..87e8f16f88c 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -160,17 +160,11 @@ public abstract class ContainerCluster<CONTAINER extends Container> private String jvmGCOptions = null; private String environmentVars = null; - private final double threadPoolSizeFactor; - private final double queueSizeFactor; - - public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId); this.name = name; this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); - this.threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); - this.queueSizeFactor = deployState.getProperties().queueSizeFactor(); componentGroup = new ComponentGroup<>(this, "component"); @@ -192,14 +186,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> addJaxProviders(); } - public double getThreadPoolSizeFactor() { - return threadPoolSizeFactor; - } - - public double getQueueSizeFactor() { - return queueSizeFactor; - } - public void setZone(Zone zone) { this.zone = zone; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java new file mode 100644 index 00000000000..c4d252ccbfe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -0,0 +1,86 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.threadpool.ContainerThreadPool; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import org.w3c.dom.Element; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Component definition for a {@link java.util.concurrent.Executor} using {@link ContainerThreadPool}. + * + * @author bjorncs + */ +public class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { + + private final String name; + private final UserOptions userOptions; + + public ContainerThreadpool(String name) { this(name, null); } + + public ContainerThreadpool(String name, UserOptions userOptions) { + super(new ComponentModel( + BundleInstantiationSpecification.getFromStrings( + "threadpool@" + name, + ContainerThreadPool.class.getName(), + null))); + this.name = name; + this.userOptions = userOptions; + } + + @Override + public void getConfig(ContainerThreadpoolConfig.Builder builder) { + builder.name(this.name); + if (userOptions != null) { + builder.maxThreads(userOptions.maxThreads); + builder.minThreads(userOptions.minThreads); + builder.queueSize(userOptions.queueSize); + } + } + + protected Optional<UserOptions> userOptions() { return Optional.ofNullable(userOptions); } + protected boolean hasUserOptions() { return userOptions().isPresent(); } + + protected static double vcpu(ContainerCluster<?> cluster) { + List<Double> vcpus = cluster.getContainers().stream() + .filter(c -> c.getHostResource() != null && c.getHostResource().realResources() != null) + .map(c -> c.getHostResource().realResources().vcpu()) + .distinct() + .collect(Collectors.toList()); + // We can only use host resource for calculation if all container nodes in the cluster are homogeneous (in terms of vcpu) + if (vcpus.size() != 1 || vcpus.get(0) == 0) return 0; + return vcpus.get(0); + } + + public static class UserOptions { + private final int maxThreads; + private final int minThreads; + private final int queueSize; + + private UserOptions(int maxThreads, int minThreads, int queueSize) { + this.maxThreads = maxThreads; + this.minThreads = minThreads; + this.queueSize = queueSize; + } + + public static Optional<UserOptions> fromXml(Element xml) { + Element element = XML.getChild(xml, "threadpool"); + if (element == null) return Optional.empty(); + return Optional.of(new UserOptions( + intOption(element, "max-threads"), + intOption(element, "min-threads"), + intOption(element, "queue-size"))); + } + + private static int intOption(Element element, String name) { + return Integer.parseInt(XML.getChild(element, name).getTextContent()); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java deleted file mode 100644 index 7eb7a1fb518..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.container.handler.ThreadpoolConfig; -import com.yahoo.search.config.QrStartConfig; - -/** - * Tuning of qr-start config for a container service based on node resources. - * - * @author balder - */ -public class NodeResourcesTuning implements QrStartConfig.Producer, ThreadpoolConfig.Producer { - - private final NodeResources resources; - - public NodeResourcesTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { - this.threadPoolSizeFactor = threadPoolSizeFactor; - return this; - } - - public NodeResourcesTuning setQueueSizeFactor(double queueSizeFactor) { - this.queueSizeFactor = queueSizeFactor; - return this; - } - - private double threadPoolSizeFactor = 8.0; - private double queueSizeFactor = 8.0; - - NodeResourcesTuning(NodeResources resources) { - this.resources = resources; - } - - @Override - public void getConfig(QrStartConfig.Builder builder) { - builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(resources.vcpu()))); - } - - @Override - public void getConfig(ThreadpoolConfig.Builder builder) { - // Controls max number of concurrent requests per container - int workerThreads = Math.max(2, (int)Math.ceil(resources.vcpu() * threadPoolSizeFactor)); - builder.maxthreads(workerThreads); - - // This controls your burst handling capability. - // 0 => No extra burst handling beyond you max concurrent requests (maxthreads). - // N => N times max concurrent requests as a buffer for handling bursts - builder.queueSize((int)(workerThreads * queueSizeFactor)); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java deleted file mode 100644 index 9d59941f603..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container; - -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.container.handler.threadpool.ContainerThreadPool; -import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; -import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.model.container.component.SimpleComponent; - -import java.time.Duration; - -/** - * Component definition for a {@link java.util.concurrent.Executor} using {@link ContainerThreadPool}. - * - * @author bjorncs - */ -public class ThreadPoolExecutorComponent extends SimpleComponent implements ContainerThreadpoolConfig.Producer { - - private final String name; - private final Integer maxPoolSize; - private final Integer corePoolSize; - private final Duration keepAliveTime; - private final Integer queueSize; - private final Duration maxThreadExecutionTime; - - private ThreadPoolExecutorComponent(Builder builder) { - super(new ComponentModel( - BundleInstantiationSpecification.getFromStrings( - "threadpool@" + builder.name, - ContainerThreadPool.class.getName(), - null))); - this.name = builder.name; - this.maxPoolSize = builder.maxPoolSize; - this.corePoolSize = builder.corePoolSize; - this.keepAliveTime = builder.keepAliveTime; - this.queueSize = builder.queueSize; - this.maxThreadExecutionTime = builder.maxThreadExecutionTime; - } - - @Override - public void getConfig(ContainerThreadpoolConfig.Builder builder) { - builder.name(this.name); - if (maxPoolSize != null) builder.maxThreads(maxPoolSize); - if (corePoolSize != null) builder.minThreads(corePoolSize); - if (keepAliveTime != null) builder.keepAliveTime(keepAliveTime.toMillis() / 1000D); - if (queueSize != null) builder.queueSize(queueSize); - if (maxThreadExecutionTime != null) builder.maxThreadExecutionTimeSeconds((int)maxThreadExecutionTime.toMillis() / 1000); - } - - public static class Builder { - - private final String name; - private Integer maxPoolSize; - private Integer corePoolSize; - private Duration keepAliveTime; - private Integer queueSize; - private Duration maxThreadExecutionTime; - - public Builder(String name) { this.name = name; } - - public Builder maxPoolSize(int size) { this.maxPoolSize = size; return this; } - public Builder corePoolSize(int size) { this.corePoolSize = size; return this; } - public Builder keepAliveTime(Duration time) { this.keepAliveTime = time; return this; } - public Builder queueSize(int size) { this.queueSize = size; return this; } - public Builder maxThreadExecutionTime(Duration time) { this.maxThreadExecutionTime = time; return this; } - - public ThreadPoolExecutorComponent build() { return new ThreadPoolExecutorComponent(this); } - - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 4e0bff1c1fc..12d74418f9f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -9,8 +9,6 @@ import org.w3c.dom.Element; import java.util.Arrays; import java.util.List; -import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; - /** * This object builds a bundle instantiation spec from an XML element. * @@ -39,7 +37,7 @@ public class BundleInstantiationSpecificationBuilder { private static void validate(BundleInstantiationSpecification instSpec) { List<String> forbiddenClasses = Arrays.asList( - SEARCH_HANDLER_CLASS, + SearchHandler.HANDLER_CLASS, "com.yahoo.processing.handler.ProcessingHandler"); for (String forbiddenClass: forbiddenClasses) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 0e77c6387ae..638c02caf55 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -54,6 +54,7 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.ContainerModelEvaluation; +import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.BindingPattern; @@ -111,9 +112,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static final String DEPRECATED_CONTAINER_TAG = "jdisc"; private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables"; - static final String SEARCH_HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); - static final BindingPattern SEARCH_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); - public enum Networking { disable, enable } private ApplicationPackage app; @@ -197,7 +195,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addClientProviders(deployState, spec, cluster); addServerProviders(deployState, spec, cluster); - addHandlerSpecificThreadpools(cluster); addAthensCopperArgos(cluster, context); // Must be added after nodes. } @@ -212,13 +209,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void addHandlerSpecificThreadpools(ContainerCluster<?> cluster) { - ContainerDocumentApi documentApi = cluster.getDocumentApi(); - if (documentApi != null) { - documentApi.addNodesDependentThreadpoolConfiguration(); - } - } - private void addAthensCopperArgos(ApplicationContainerCluster cluster, ConfigModelContext context) { if ( ! context.getDeployState().isHosted()) return; app.getDeployment().map(DeploymentSpec::fromXml) @@ -427,7 +417,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addIncludes(searchElement); cluster.setSearch(buildSearch(deployState, cluster, searchElement)); - addSearchHandler(cluster, searchElement); + addSearchHandler(cluster, searchElement, deployState); addGUIHandler(cluster); validateAndAddConfiguredComponents(deployState, cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -447,7 +437,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), - serverBindings(processingElement, ProcessingChains.defaultBindings)); + serverBindings(processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); validateAndAddConfiguredComponents(deployState, cluster, processingElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -791,19 +781,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { container.setPreLoad(nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)); } - private void addSearchHandler(ApplicationContainerCluster cluster, Element searchElement) { + private void addSearchHandler(ApplicationContainerCluster cluster, Element searchElement, DeployState deployState) { // Magic spell is needed to receive the chains config :-| cluster.addComponent(new ProcessingHandler<>(cluster.getSearch().getChains(), "com.yahoo.search.searchchain.ExecutionFactory")); - ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(cluster.getSearch().getChains(), - "com.yahoo.search.handler.SearchHandler"); - BindingPattern[] defaultBindings = {SEARCH_HANDLER_BINDING}; - for (BindingPattern binding: serverBindings(searchElement, defaultBindings)) { - searchHandler.addServerBindings(binding); - } - - cluster.addComponent(searchHandler); + cluster.addComponent( + new SearchHandler( + cluster, + serverBindings(searchElement, SearchHandler.DEFAULT_BINDING), + ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null), + deployState)); } private void addGUIHandler(ApplicationContainerCluster cluster) { @@ -813,15 +801,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } - private BindingPattern[] serverBindings(Element searchElement, BindingPattern... defaultBindings) { + private List<BindingPattern> serverBindings(Element searchElement, BindingPattern... defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) - return defaultBindings; + return List.of(defaultBindings); return toBindingList(bindings); } - private BindingPattern[] toBindingList(List<Element> bindingElements) { + private List<BindingPattern> toBindingList(List<Element> bindingElements) { List<BindingPattern> result = new ArrayList<>(); for (Element element: bindingElements) { @@ -830,7 +818,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { result.add(UserBindingPattern.fromPattern(text)); } - return result.toArray(BindingPattern[]::new); + return result; } private ContainerDocumentApi buildDocumentApi(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index 34de21de404..99ae6184f5c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.XML; import com.yahoo.vespa.model.clients.ContainerDocumentApi; +import com.yahoo.vespa.model.container.ContainerThreadpool; import org.w3c.dom.Element; import java.util.ArrayList; @@ -20,7 +21,17 @@ public class DocumentApiOptionsBuilder { public static ContainerDocumentApi.Options build(DeployState deployState, Element spec) { - return new ContainerDocumentApi.Options(getBindings(spec), deployState.getProperties().feedCoreThreadPoolSizeFactor()); + return new ContainerDocumentApi.Options( + getBindings(spec), + threadpoolOptions(spec, "rest-api"), + threadpoolOptions(spec, "http-client-api"), + deployState.getProperties().feedCoreThreadPoolSizeFactor()); + } + + private static ContainerThreadpool.UserOptions threadpoolOptions(Element spec, String elementName) { + Element element = XML.getChild(spec, elementName); + if (element == null) return null; + return ContainerThreadpool.UserOptions.fromXml(element).orElse(null); } private static List<String> getBindings(Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java new file mode 100644 index 00000000000..81ab2cc1503 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.ContainerThreadpool; +import com.yahoo.vespa.model.container.component.BindingPattern; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; + +import java.util.List; + +/** + * Component definition for {@link com.yahoo.search.handler.SearchHandler} + * + * @author bjorncs + */ +class SearchHandler extends ProcessingHandler<SearchChains> { + + static final String HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); + static final BindingPattern DEFAULT_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); + + private final ApplicationContainerCluster cluster; + + SearchHandler(ApplicationContainerCluster cluster, + List<BindingPattern> bindings, + ContainerThreadpool.UserOptions threadpoolOptions, + DeployState deployState) { + super(cluster.getSearchChains(), HANDLER_CLASS); + this.cluster = cluster; + bindings.forEach(this::addServerBindings); + Threadpool threadpool = new Threadpool(cluster, threadpoolOptions, deployState); + inject(threadpool); + addComponent(threadpool); + } + + private static class Threadpool extends ContainerThreadpool { + private final ApplicationContainerCluster cluster; + private final DeployState deployState; + + Threadpool(ApplicationContainerCluster cluster, UserOptions options, DeployState deployState) { + super("search-handler", options); + this.cluster = cluster; + this.deployState = deployState; + } + + @Override + public void getConfig(ContainerThreadpoolConfig.Builder builder) { + super.getConfig(builder); + + builder.maxThreadExecutionTimeSeconds(190); + builder.keepAliveTime(5.0); + + // User options overrides below configuration + if (hasUserOptions()) return; + + double threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); + double vcpu = vcpu(cluster); + if (threadPoolSizeFactor <= 0 || vcpu == 0) { + builder.maxThreads(500); + builder.minThreads(500); + builder.queueSize(0); + } else { + // Controls max number of concurrent requests per container + int workerThreads = Math.max(2, (int)Math.ceil(vcpu * threadPoolSizeFactor)); + builder.maxThreads(workerThreads); + builder.minThreads(workerThreads); + + // This controls your burst handling capability. + // 0 => No extra burst handling beyond you max concurrent requests (maxthreads). + // N => N times max concurrent requests as a buffer for handling bursts + builder.queueSize((int)(workerThreads * deployState.getProperties().queueSizeFactor())); + } + } + + + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index f2e90ae2859..34b6dd017cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -27,6 +27,7 @@ public abstract class ContentNode extends AbstractService private final boolean skipCommunicationManagerThread; private final boolean skipMbusRequestThread; private final boolean skipMbusReplyThread; + private final boolean useDirectStorageApiRpc; public ContentNode(ModelContext.Properties properties, AbstractConfigProducer parent, String clusterName, String rootDirectory, int distributionKey) { super(parent, "" + distributionKey); @@ -35,6 +36,7 @@ public abstract class ContentNode extends AbstractService this.skipMbusRequestThread = properties.skipMbusRequestThread(); this.skipMbusReplyThread = properties.skipMbusReplyThread(); this.rootDirectory = rootDirectory; + this.useDirectStorageApiRpc = properties.useDirectStorageApiRpc(); initialize(); setProp("clustertype", "content"); @@ -81,6 +83,7 @@ public abstract class ContentNode extends AbstractService builder.skip_thread(skipCommunicationManagerThread); builder.mbus.skip_request_thread(skipMbusRequestThread); builder.mbus.skip_reply_thread(skipMbusReplyThread); + builder.use_direct_storageapi_rpc(useDirectStorageApiRpc); } @Override diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 3c8b60fb84b..98ea696ceef 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -105,6 +105,12 @@ SslProvider = element ssl-provider { BundleSpec } +Threadpool = element threadpool { + element max-threads { xsd:nonNegativeInteger } & + element min-threads { xsd:nonNegativeInteger } & + element queue-size { xsd:nonNegativeInteger } +} + # REST-API: RestApi = element rest-api { @@ -142,7 +148,8 @@ SearchInContainer = element search { SearchChain* & Provider* & Renderer* & - GenericConfig* + GenericConfig* & + Threadpool? } SearchChain = element chain { @@ -207,10 +214,18 @@ DocumentApi = element document-api { element retrydelay { xsd:double { minInclusive = "0.0" } }? & element timeout { xsd:double { minInclusive = "0.0" } }? & element tracelevel { xsd:positiveInteger }? & - element mbusport { xsd:positiveInteger }? + element mbusport { xsd:positiveInteger }? & + DocumentRestApi? & + HttpClientApi? } +DocumentRestApi = element rest-api { + Threadpool? +} +HttpClientApi = element http-client-api { + Threadpool? +} # NODES: diff --git a/config-model/src/main/resources/schema/federation.rnc b/config-model/src/main/resources/schema/federation.rnc index 6dbeb32fbb2..227bebc16ba 100644 --- a/config-model/src/main/resources/schema/federation.rnc +++ b/config-model/src/main/resources/schema/federation.rnc @@ -16,6 +16,8 @@ Provider = attribute cachesize { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & attribute type { xsd:string }? & attribute cluster { xsd:string }? & + + # TODO Vespa 8 Remove yca concepts from services.xml syntax attribute yca-application-id { xsd:string }? & attribute yca-cache-ttl { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & attribute yca-cache-retry-wait { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index be7fc19a429..71dc1564277 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -87,7 +87,7 @@ public class DedicatedAdminV4Test { MetricsConsumer consumer = model.getAdmin().getUserMetrics().getConsumers().get("slingstone"); assertNotNull(consumer); - Metric metric = consumer.getMetrics().get("foobar.count"); + Metric metric = consumer.metrics().get("foobar.count"); assertNotNull(metric); assertEquals("foobar", metric.outputName); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java index 12a10a7e354..36eb30073b3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -4,6 +4,7 @@ import ai.vespa.metricsproxy.core.ConsumersConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricSet; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -16,13 +17,11 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.c import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -34,7 +33,7 @@ import static org.junit.Assert.assertTrue; */ public class MetricsConsumersTest { - private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size(); + private static int numPublicDefaultMetrics = defaultMetricSet.getMetrics().size(); private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); private static int numSystemMetrics = systemMetricSet.getMetrics().size(); @@ -48,9 +47,9 @@ public class MetricsConsumersTest { public void default_public_consumer_is_set_up_for_self_hosted() { ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + assertEquals(MetricsConsumer.defaultConsumer.id(), config.consumer(1).name()); - int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics; + int numMetricsForPublicDefaultConsumer = defaultMetricSet.getMetrics().size() + numSystemMetrics; assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size()); } @@ -58,14 +57,14 @@ public class MetricsConsumersTest { public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() { ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + assertEquals(MetricsConsumer.vespa.id(), config.consumer(0).name()); + assertEquals(MetricsConsumer.defaultConsumer.id(), config.consumer(1).name()); } @Test public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); + assertEquals(MetricsConsumer.vespa.id(), config.consumer(0).name()); assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index 8ecb13d7ae5..b6037d2614e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -14,12 +14,11 @@ import ai.vespa.metricsproxy.service.VespaServicesConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.monitoring.Metric; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.test.VespaModelTester; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; /** * @author gjoranv @@ -76,7 +75,8 @@ class MetricsProxyModelTester { static ConsumersConfig.Consumer getCustomConsumer(String servicesXml) { ConsumersConfig config = consumersConfigFromXml(servicesXml, self_hosted); for (ConsumersConfig.Consumer consumer : config.consumer()) { - if (! consumer.name().equals(VESPA_CONSUMER_ID) && ! consumer.name().equals(DEFAULT_PUBLIC_CONSUMER_ID)) + if (! consumer.name().equals(MetricsConsumer.vespa.id()) && + ! consumer.name().equals(MetricsConsumer.defaultConsumer.id())) return consumer; } throw new RuntimeException("Custom consumer not found!"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 97359b392a5..d493afd9c1f 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -10,11 +10,9 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; @@ -230,20 +228,6 @@ public class ContainerClusterTest { } @Test - public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() { - FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3); - NodeResourcesTuning nodeResourcesTuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources()) - .setThreadPoolSizeFactor(13.3) - .setQueueSizeFactor(17.5); - - ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - nodeResourcesTuning.getConfig(tpBuilder); - ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); - assertEquals(40, threadpoolConfig.maxthreads()); - assertEquals(700, threadpoolConfig.queueSize()); - } - - @Test public void requireThatDefaultThreadPoolConfigIsSane() { MockRoot root = new MockRoot("foo"); ApplicationContainerCluster cluster = createContainerCluster(root, false); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index 93117821c5a..def5da3a9c2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -121,6 +121,43 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertEquals(8, config.minThreads()); } + @Test + public void threadpools_configuration_can_be_overridden() { + Element elem = DomBuilderTest.parse( + "<container id='cluster1' version='1.0'>", + " <document-api>", + " <rest-api>", + " <threadpool>", + " <max-threads>20</max-threads>", + " <min-threads>10</min-threads>", + " <queue-size>0</queue-size>", + " </threadpool>", + " </rest-api>", + " <http-client-api>", + " <threadpool>", + " <max-threads>50</max-threads>", + " <min-threads>25</min-threads>", + " <queue-size>1000</queue-size>", + " </threadpool>", + " </http-client-api>", + " </document-api>", + nodesXml, + "</container>"); + createModel(root, elem); + + ContainerThreadpoolConfig restApiThreadpoolConfig = root.getConfig( + ContainerThreadpoolConfig.class, "cluster1/component/com.yahoo.document.restapi.resource.RestApi/threadpool@restapi-handler"); + assertEquals(20, restApiThreadpoolConfig.maxThreads()); + assertEquals(10, restApiThreadpoolConfig.minThreads()); + assertEquals(0, restApiThreadpoolConfig.queueSize()); + + ContainerThreadpoolConfig feedThreadpoolConfig = root.getConfig( + ContainerThreadpoolConfig.class, "cluster1/component/com.yahoo.vespa.http.server.FeedHandler/threadpool@feedapi-handler"); + assertEquals(50, feedThreadpoolConfig.maxThreads()); + assertEquals(25, feedThreadpoolConfig.minThreads()); + assertEquals(1000, feedThreadpoolConfig.queueSize()); + } + private static class HostProvisionerWithCustomRealResource implements HostProvisioner { @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index c8564c5a273..4c1fda44038 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -4,10 +4,11 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.container.core.ChainsConfig; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.search.GUIHandler; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; @@ -18,9 +19,8 @@ import org.w3c.dom.Element; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static com.yahoo.test.Matchers.hasItemWithMethod; import static com.yahoo.vespa.model.container.search.ContainerSearch.QUERY_PROFILE_REGISTRY_CLASS; -import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING; -import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -105,7 +105,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { "<container id='default' version='1.0'>", " <search />", " <handler id='" + myHandler + "'>", - " <binding>" + SEARCH_HANDLER_BINDING.patternString() + "</binding>", + " <binding>" + SearchHandler.DEFAULT_BINDING.patternString() + "</binding>", " </handler>", nodesXml, "</container>"); @@ -113,8 +113,8 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); - assertEquals(SEARCH_HANDLER_BINDING.patternString(), discBindingsConfig.handlers(myHandler).serverBindings(0)); - assertNull(discBindingsConfig.handlers(SEARCH_HANDLER_CLASS)); + assertEquals(SearchHandler.DEFAULT_BINDING.patternString(), discBindingsConfig.handlers(myHandler).serverBindings(0)); + assertNull(discBindingsConfig.handlers(SearchHandler.HANDLER_CLASS)); } // TODO: remove test when all containers are named 'container' @@ -222,6 +222,51 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { assertFalse(cluster.getSearchChains().localProviders().isEmpty()); } + @Test + public void search_handler_has_dedicated_threadpool() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <search />", + nodesXml, + "</container>"); + + createModel(root, clusterElem); + + ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); + Handler<?> searchHandler = cluster.getHandlers().stream() + .filter(h -> h.getComponentId().toString().equals(SearchHandler.HANDLER_CLASS)) + .findAny() + .get(); + + assertThat(searchHandler.getInjectedComponentIds(), hasItem("threadpool@search-handler")); + + ContainerThreadpoolConfig config = root.getConfig( + ContainerThreadpoolConfig.class, "default/component/" + SearchHandler.HANDLER_CLASS + "/threadpool@search-handler"); + assertEquals(500, config.maxThreads()); + assertEquals(500, config.minThreads()); + assertEquals(0, config.queueSize()); + } + + @Test + public void threadpool_configuration_can_be_overridden() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <search>", + " <threadpool>", + " <max-threads>100</max-threads>", + " <min-threads>80</min-threads>", + " <queue-size>10</queue-size>", + " </threadpool>", + " </search>", + nodesXml, + "</container>"); + createModel(root, clusterElem); + ContainerThreadpoolConfig config = root.getConfig( + ContainerThreadpoolConfig.class, "default/component/" + SearchHandler.HANDLER_CLASS + "/threadpool@search-handler"); + assertEquals(100, config.maxThreads()); + assertEquals(80, config.minThreads()); + assertEquals(10, config.queueSize()); + } private VespaModel getVespaModelWithMusic(String hosts, String services) { return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSchemas("music")).create(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 27c88ad2d1f..61f5ec56bb4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; +import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.search.DispatchConfig; @@ -980,4 +981,31 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveThreePhaseUpdateConfigWithFeatureFlag(true)); } + void assertDirectStorageApiRpcConfig(boolean expUseDirectStorageApiRpc, ContentNode node) { + var builder = new StorCommunicationmanagerConfig.Builder(); + node.getConfig(builder); + var config = new StorCommunicationmanagerConfig(builder); + assertEquals(expUseDirectStorageApiRpc, config.use_direct_storageapi_rpc()); + } + + void assertDirectStorageApiRpcFlagIsPropagatedToConfig(boolean useDirectStorageApiRpc) { + VespaModel model = createEnd2EndOneNode(new TestProperties().setUseDirectStorageApiRpc(useDirectStorageApiRpc)); + + ContentCluster cc = model.getContentClusters().get("storage"); + assertFalse(cc.getDistributorNodes().getChildren().isEmpty()); + for (Distributor d : cc.getDistributorNodes().getChildren().values()) { + assertDirectStorageApiRpcConfig(useDirectStorageApiRpc, d); + } + assertFalse(cc.getStorageNodes().getChildren().isEmpty()); + for (StorageNode node : cc.getStorageNodes().getChildren().values()) { + assertDirectStorageApiRpcConfig(useDirectStorageApiRpc, node); + } + } + + @Test + public void use_direct_storage_api_rpc_config_is_controlled_by_properties() { + assertDirectStorageApiRpcFlagIsPropagatedToConfig(false); + assertDirectStorageApiRpcFlagIsPropagatedToConfig(true); + } + } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index e7ea2683e3f..683e2dc0b0d 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -158,6 +158,20 @@ <timeout>5.55</timeout> <route>default</route> <maxpendingdocs>100</maxpendingdocs> + <rest-api> + <threadpool> + <max-threads>50</max-threads> + <min-threads>10</min-threads> + <queue-size>1000</queue-size> + </threadpool> + </rest-api> + <http-client-api> + <threadpool> + <max-threads>50</max-threads> + <min-threads>10</min-threads> + <queue-size>1000</queue-size> + </threadpool> + </http-client-api> </document-api> <search> @@ -184,6 +198,12 @@ </chain> <chain id="achain" searchers="asearcher anothersearcher" inherits="wonkaparentchain" excludes="notneededsearcher"/> + + <threadpool> + <max-threads>500</max-threads> + <min-threads>500</min-threads> + <queue-size>0</queue-size> + </threadpool> </search> <processing> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java index 2e53d4384a4..9cfe3fdd1cc 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java @@ -65,4 +65,7 @@ public interface Deployer { /** Returns the time the current local active session was created, or empty if there is no local active session */ Optional<Instant> lastDeployTime(ApplicationId application); + /** Whether the deployer is bootstrapping, some users of the deployer will want to hold off with deployments in that case. */ + default boolean bootstrapping() { return false; }; + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index a19d5809660..30f4884c737 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -60,13 +60,13 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer } void shutdown() { - supervisor.transport().shutdown(); try { rpcExecutor.shutdownNow(); rpcExecutor.awaitTermination(10, TimeUnit.SECONDS); } catch (InterruptedException e) { throw new RuntimeException(e); } + supervisor.transport().shutdown(); } Spec getSpec() { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 75866ec73ab..0085758f325 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -51,12 +51,12 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { private final static TimingValues timingValues; private final ScheduledExecutorService nextConfigScheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory("next config")); - private ScheduledFuture<?> nextConfigFuture; + private final ScheduledFuture<?> nextConfigFuture; private final JRTConfigRequester requester; // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients private final ScheduledExecutorService delayedResponsesScheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory("delayed responses")); - private ScheduledFuture<?> delayedResponsesFuture; + private final ScheduledFuture<?> delayedResponsesFuture; static { // Proxy should time out before clients upon subscription. @@ -181,9 +181,10 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { public void cancel() { shutdownSourceConnections(); delayedResponsesFuture.cancel(true); - delayedResponsesScheduler.shutdown(); + delayedResponsesScheduler.shutdownNow(); nextConfigFuture.cancel(true); - nextConfigScheduler.shutdown(); + nextConfigScheduler.shutdownNow(); + requester.close(); } /** @@ -193,7 +194,7 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { public void shutdownSourceConnections() { activeSubscribers.values().forEach(Subscriber::cancel); activeSubscribers.clear(); - nextConfigScheduler.shutdown(); + nextConfigScheduler.shutdownNow(); requester.close(); } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java index f4858843574..2f3a4bd2172 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java @@ -22,6 +22,7 @@ import java.io.OutputStream; * @author Ulf Lilleengen */ public class ConfigPayload { + private final Slime slime; public ConfigPayload(Slime slime) { diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 1bab29e727e..329173d6f8c 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -120,8 +120,7 @@ main(int argc, char **argv) lastTv = tv; } - int rv = handler.terminate(); - EV_STOPPING("config-sentinel", "normal exit"); + int rv = handler.terminate(); return rv; } diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 0673e803057..e1ca58110fe 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -62,8 +62,6 @@ vespa_generate_config(configdefinitions summarymap.def) install_config_definition(summarymap.def vespa.config.search.summarymap.def) vespa_generate_config(configdefinitions upgrading.def) install_config_definition(upgrading.def vespa.config.content.upgrading.def) -vespa_generate_config(configdefinitions ymon.def) -install_config_definition(ymon.def cloud.config.ymon.def) vespa_generate_config(configdefinitions zookeeper-server.def) install_config_definition(zookeeper-server.def cloud.config.zookeeper-server.def) vespa_generate_config(configdefinitions zookeepers.def) diff --git a/configdefinitions/src/vespa/ymon.def b/configdefinitions/src/vespa/ymon.def deleted file mode 100644 index 2296248bf8b..00000000000 --- a/configdefinitions/src/vespa/ymon.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -services[].servicetype string default="(unknownservicetype)" -services[].ymonname string default="(unknownymonname)" -services[].hosts[] string diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index fb07bf626f3..3564a6e6da7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -87,6 +87,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -112,6 +113,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private static final Logger log = Logger.getLogger(ApplicationRepository.class.getName()); + private final AtomicBoolean bootstrapping = new AtomicBoolean(true); + private final TenantRepository tenantRepository; private final Optional<Provisioner> hostProvisioner; private final Optional<InfraDeployer> infraDeployer; @@ -268,6 +271,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Deploying ---------------------------------------------------------------- + @Override + public boolean bootstrapping() { + return bootstrapping.get(); + } + + public void bootstrappingDone() { + bootstrapping.set(false); + } + public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, Instant now) { validateThatLocalSessionIsNotActive(tenant, sessionId); LocalSession session = getLocalSession(tenant, sessionId); @@ -819,7 +831,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public void deleteExpiredLocalSessions() { - Map<Tenant, List<LocalSession>> sessionsPerTenant = new HashMap<>(); + Map<Tenant, Collection<LocalSession>> sessionsPerTenant = new HashMap<>(); tenantRepository.getAllTenants().forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getSessionRepository().getLocalSessions())); Set<ApplicationId> applicationIds = new HashSet<>(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 749561d5fdb..609ff4473c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -10,11 +10,8 @@ import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.TransientException; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.state.StateMonitor; -import com.yahoo.vespa.config.server.maintenance.FileDistributionMaintainer; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -166,6 +163,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable return; // Status will not be set to 'up' since we return here } } + applicationRepository.bootstrappingDone(); startRpcServer(); up(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java index 659662ebcf6..fa30fa3561a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java @@ -25,6 +25,10 @@ public class TimeoutBudget { this.endTime = startTime.plus(duration); } + public Duration timeout() { + return Duration.between(startTime, endTime); + } + public Duration timeLeft() { Instant now = clock.instant(); measurements.add(new Measurement(now)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index 51213b173dd..2e73a02c75b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -37,7 +37,9 @@ import java.util.Set; public class Application implements ModelResult { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Application.class.getName()); - private final long appGeneration; // The config generation for this application + + /** The config generation for this application. */ + private final long applicationGeneration; private final boolean internalRedeploy; private final Version vespaVersion; private final Model model; @@ -45,12 +47,12 @@ public class Application implements ModelResult { private final MetricUpdater metricUpdater; private final ApplicationId app; - public Application(Model model, ServerCache cache, long appGeneration, boolean internalRedeploy, + public Application(Model model, ServerCache cache, long applicationGeneration, boolean internalRedeploy, Version vespaVersion, MetricUpdater metricUpdater, ApplicationId app) { Objects.requireNonNull(model, "The model cannot be null"); this.model = model; this.cache = cache; - this.appGeneration = appGeneration; + this.applicationGeneration = applicationGeneration; this.internalRedeploy = internalRedeploy; this.vespaVersion = vespaVersion; this.metricUpdater = metricUpdater; @@ -62,7 +64,7 @@ public class Application implements ModelResult { * * @return the config generation */ - public Long getApplicationGeneration() { return appGeneration; } + public Long getApplicationGeneration() { return applicationGeneration; } /** Returns the application model, never null */ @Override @@ -72,13 +74,13 @@ public class Application implements ModelResult { public String toString() { StringBuilder sb = new StringBuilder(); sb.append("application '").append(app.application().value()).append("', "); - sb.append("generation ").append(appGeneration).append(", "); + sb.append("generation ").append(applicationGeneration).append(", "); sb.append("vespa version ").append(vespaVersion); return sb.toString(); } public ApplicationInfo toApplicationInfo() { - return new ApplicationInfo(app, appGeneration, model); + return new ApplicationInfo(app, applicationGeneration, model); } public ServerCache getCache() { @@ -134,7 +136,7 @@ public class Application implements ModelResult { throw new ConfigurationRuntimeException("Unable to resolve config " + configKey); } - ConfigResponse configResponse = responseFactory.createResponse(payload, appGeneration, internalRedeploy); + ConfigResponse configResponse = responseFactory.createResponse(payload, applicationGeneration, internalRedeploy); metricUpdater.incrementProcTime(System.currentTimeMillis() - start); if (useCache(req)) { cache.put(cacheKey, configResponse, configResponse.getConfigMd5()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 3d4198c65a9..87b0ed965d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -149,6 +149,7 @@ public class ModelContextImpl implements ModelContext { private final boolean isFirstTimeDeployment; private final boolean useContentNodeBtreeDb; private final boolean useThreePhaseUpdates; + private final boolean useDirectStorageApiRpc; private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final double threadPoolSizeFactor; @@ -199,6 +200,8 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); useThreePhaseUpdates = Flags.USE_THREE_PHASE_UPDATES.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + useDirectStorageApiRpc = Flags.USE_DIRECT_STORAGE_API_RPC.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) @@ -288,6 +291,11 @@ public class ModelContextImpl implements ModelContext { } @Override + public boolean useDirectStorageApiRpc() { + return useDirectStorageApiRpc; + } + + @Override public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java index a1a41cc0472..dac9acb42bd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java @@ -4,9 +4,15 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; import static com.yahoo.jdisc.http.HttpResponse.Status.OK; @@ -17,6 +23,8 @@ import static com.yahoo.jdisc.http.HttpResponse.Status.OK; */ public class SessionContentReadResponse extends HttpResponse { + private static final Map<String, String> contentTypeByExtension = loadContentTypeByExtension(); + private final ApplicationFile file; public SessionContentReadResponse(ApplicationFile file) { @@ -33,7 +41,34 @@ public class SessionContentReadResponse extends HttpResponse { @Override public String getContentType() { - return HttpResponse.DEFAULT_MIME_TYPE; + String filename = file.getPath().getName(); + int lastDotIndex = filename.lastIndexOf('.'); + if (lastDotIndex >= 0) { + String contentType = contentTypeByExtension.get(filename.substring(lastDotIndex + 1)); + if (contentType != null) return contentType; + } + return DEFAULT_MIME_TYPE; + } + + private static Map<String, String> loadContentTypeByExtension() { + ClassLoader classLoader = SessionContentReadResponse.class.getClassLoader(); + Pattern whitespace = Pattern.compile("\\s"); + Map<String, String> map = new HashMap<>(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(classLoader.getResourceAsStream("mime.types")))) { + while (reader.ready()) { + String line = reader.readLine(); + if (line.isEmpty() || line.charAt(0) == '#') continue; + + String[] parts = whitespace.split(line); + for (int i = 1; i < parts.length; i++) + map.putIfAbsent(parts[i], parts[0]); + } + + return map; + } catch (IOException e) { + throw new UncheckedIOException(e); + } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index ed526908f6f..0be81e41d30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -28,12 +28,12 @@ public class SessionsMaintainer extends ConfigServerMaintainer { applicationRepository.deleteExpiredLocalSessions(); if (hostedVespa) { - Duration expiryTime = Duration.ofHours(2); + Duration expiryTime = Duration.ofMinutes(90); int deleted = applicationRepository.deleteExpiredRemoteSessions(expiryTime); log.log(LogLevel.FINE, () -> "Deleted " + deleted + " expired remote sessions older than " + expiryTime); } - Duration lockExpiryTime = Duration.ofHours(2); + Duration lockExpiryTime = Duration.ofMinutes(90); int deleted = applicationRepository.deleteExpiredSessionLocks(lockExpiryTime); log.log(LogLevel.FINE, () -> "Deleted " + deleted + " locks older than " + lockExpiryTime); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 6d2ef4028c6..7601d90043e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -50,7 +50,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private static final Logger log = Logger.getLogger(ActivatedModelsBuilder.class.getName()); private final TenantName tenant; - private final long appGeneration; + private final long applicationGeneration; private final SessionZooKeeperClient zkClient; private final PermanentApplicationPackage permanentApplicationPackage; private final ConfigDefinitionRepo configDefinitionRepo; @@ -60,7 +60,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final SecretStore secretStore; public ActivatedModelsBuilder(TenantName tenant, - long appGeneration, + long applicationGeneration, SessionZooKeeperClient zkClient, GlobalComponentRegistry globalComponentRegistry) { super(globalComponentRegistry.getModelFactoryRegistry(), @@ -68,7 +68,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { globalComponentRegistry.getZone(), HostProvisionerProvider.from(globalComponentRegistry.getHostProvisioner())); this.tenant = tenant; - this.appGeneration = appGeneration; + this.applicationGeneration = applicationGeneration; this.zkClient = zkClient; this.permanentApplicationPackage = globalComponentRegistry.getPermanentApplicationPackage(); this.configDefinitionRepo = globalComponentRegistry.getStaticConfigDefinitionRepo(); @@ -87,7 +87,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { Optional<AllocatedHosts> ignored // Ignored since we have this in the app package for activated models ) { log.log(Level.FINE, String.format("Loading model version %s for session %s application %s", - modelFactory.version(), appGeneration, applicationId)); + modelFactory.version(), applicationGeneration, applicationId)); ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( @@ -110,7 +110,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { ServerCache serverCache = new ServerCache(configDefinitionRepo, zkClient.getUserConfigDefinitions()); return new Application(modelFactory.createModel(modelContext), serverCache, - appGeneration, + applicationGeneration, applicationPackage.getMetaData().isInternalRedeploy(), modelFactory.version(), applicationMetricUpdater, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java index 88aa41ca735..415fa764823 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/ConfigResponseFactory.java @@ -26,10 +26,11 @@ public interface ConfigResponseFactory { /** * Creates a {@link ConfigResponse} for a given payload and generation. - * @param payload the {@link ConfigPayload} to put in the response. - * @param generation the payload generation. @return A {@link ConfigResponse} that can be sent to the client. - * @param internalRedeploy whether this config generation was produced by an internal redeployment, + * @param payload the {@link ConfigPayload} to put in the response + * @param generation the payload generation + * @param internalRedeploy whether this config generation was produced by an internal redeployment * not a change to the application package + * @return a {@link ConfigResponse} that can be sent to the client */ ConfigResponse createResponse(ConfigPayload payload, long generation, boolean internalRedeploy); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index b3e35e955de..b389169055e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -117,9 +117,7 @@ public abstract class Session implements Comparable<Session> { } void setApplicationPackageReference(FileReference applicationPackageReference) { - if (applicationPackageReference == null) throw new IllegalArgumentException(String.format( - "Null application package file reference for tenant %s, session id %d", tenant, sessionId)); - sessionZooKeeperClient.writeApplicationPackageReference(applicationPackageReference); + sessionZooKeeperClient.writeApplicationPackageReference(Optional.ofNullable(applicationPackageReference)); } public void setVespaVersion(Version version) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java deleted file mode 100644 index 60fa037e99a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * A session cache that can store any type of {@link Session}. - * - * @author Ulf Lilleengen - * @author hmusum - */ -public class SessionCache<SESSIONTYPE extends Session> { - - private final HashMap<Long, SESSIONTYPE> sessions = new HashMap<>(); - - public synchronized void putSession(SESSIONTYPE session) { - sessions.put(session.getSessionId(), session); - } - - synchronized void removeSession(long id) { - sessions.remove(id); - } - - public synchronized SESSIONTYPE getSession(long id) { - return sessions.get(id); - } - - public synchronized List<SESSIONTYPE> getSessions() { - return new ArrayList<>(sessions.values()); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 489cb8c8167..a16d3d0c221 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -28,6 +28,7 @@ import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.lang.SettableOptional; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; @@ -234,14 +235,16 @@ public class SessionPreparer { } void checkTimeout(String step) { - if (! params.getTimeoutBudget().hasTimeLeft(step)) { - String used = params.getTimeoutBudget().timesUsed(); - throw new RuntimeException("prepare timed out " + used + " after " + step + " step: " + applicationId); + TimeoutBudget timeoutBudget = params.getTimeoutBudget(); + if (! timeoutBudget.hasTimeLeft(step)) { + String used = timeoutBudget.timesUsed(); + throw new RuntimeException("prepare timed out " + used + " after " + step + + " step (timeout " + timeoutBudget.timeout() + "): " + applicationId); } } - FileReference distributeApplicationPackage() { - if ( ! distributeApplicationPackage.value()) return null; + Optional<FileReference> distributeApplicationPackage() { + if ( ! distributeApplicationPackage.value()) return Optional.empty(); FileRegistry fileRegistry = fileDistributionProvider.getFileRegistry(); FileReference fileReference = fileRegistry.addApplicationPackage(); @@ -252,7 +255,7 @@ public class SessionPreparer { .forEach(spec -> fileDistribution.startDownload(spec.getHostName(), spec.getConfigServerPort(), Set.of(fileReference))); checkTimeout("distributeApplicationPackage"); - return fileReference; + return Optional.of(fileReference); } void preprocess() { @@ -277,7 +280,7 @@ public class SessionPreparer { checkTimeout("making result from models"); } - void writeStateZK(FileReference distributedApplicationPackage) { + void writeStateZK(Optional<FileReference> distributedApplicationPackage) { log.log(Level.FINE, "Writing application package state to zookeeper"); writeStateToZooKeeper(sessionZooKeeperClient, preprocessedApplicationPackage, @@ -332,7 +335,7 @@ public class SessionPreparer { private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, ApplicationPackage applicationPackage, ApplicationId applicationId, - FileReference distributedApplicationPackage, + Optional<FileReference> distributedApplicationPackage, Optional<DockerImage> dockerImageRepository, Version vespaVersion, DeployLogger deployLogger, @@ -343,7 +346,7 @@ public class SessionPreparer { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); - // Note: When changing the below you need to also change similar calls in SessionFactoryImpl.createSessionFromExisting() + // Note: When changing the below you need to also change similar calls in SessionRepository.createSessionFromExisting() zooKeeperClient.writeApplicationId(applicationId); if (distributeApplicationPackage.value()) zooKeeperClient.writeApplicationPackageReference(distributedApplicationPackage); zooKeeperClient.writeVespaVersion(vespaVersion); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index a9bab3ffdf8..4f577d8f62c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -46,11 +46,13 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.logging.Level; import java.util.logging.Logger; @@ -71,8 +73,8 @@ public class SessionRepository { private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); private static final long nonExistingActiveSessionId = 0; - private final SessionCache<LocalSession> localSessionCache = new SessionCache<>(); - private final SessionCache<RemoteSession> remoteSessionCache = new SessionCache<>(); + private final Map<Long, LocalSession> localSessionCache = new ConcurrentHashMap<>(); + private final Map<Long, RemoteSession> remoteSessionCache = new ConcurrentHashMap<>(); private final Map<Long, SessionStateWatcher> sessionStateWatchers = new HashMap<>(); private final Duration sessionLifetime; private final Clock clock; @@ -121,18 +123,18 @@ public class SessionRepository { // ---------------- Local sessions ---------------------------------------------------------------- public synchronized void addLocalSession(LocalSession session) { - localSessionCache.putSession(session); + localSessionCache.put(session.getSessionId(), session); long sessionId = session.getSessionId(); RemoteSession remoteSession = createRemoteSession(sessionId); addSessionStateWatcher(sessionId, remoteSession); } public LocalSession getLocalSession(long sessionId) { - return localSessionCache.getSession(sessionId); + return localSessionCache.get(sessionId); } - public List<LocalSession> getLocalSessions() { - return localSessionCache.getSessions(); + public Collection<LocalSession> getLocalSessions() { + return localSessionCache.values(); } private void loadLocalSessions() { @@ -173,7 +175,7 @@ public class SessionRepository { public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { log.log(Level.FINE, () -> "Purging old sessions for tenant '" + tenantName + "'"); try { - for (LocalSession candidate : localSessionCache.getSessions()) { + for (LocalSession candidate : localSessionCache.values()) { Instant createTime = candidate.getCreateTime(); log.log(Level.FINE, () -> "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); @@ -207,13 +209,14 @@ public class SessionRepository { return candidate.getStatus() == Session.Status.ACTIVATE; } + // Will delete session data in ZooKeeper and file system public void deleteLocalSession(LocalSession session) { long sessionId = session.getSessionId(); try (Lock lock = lock(sessionId)) { log.log(Level.FINE, () -> "Deleting local session " + sessionId); SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); if (watcher != null) watcher.close(); - localSessionCache.removeSession(sessionId); + localSessionCache.remove(sessionId); deletePersistentData(sessionId); } } @@ -246,7 +249,7 @@ public class SessionRepository { } private void deleteAllSessions() { - List<LocalSession> sessions = new ArrayList<>(localSessionCache.getSessions()); + List<LocalSession> sessions = new ArrayList<>(localSessionCache.values()); for (LocalSession session : sessions) { deleteLocalSession(session); } @@ -255,7 +258,7 @@ public class SessionRepository { // ---------------- Remote sessions ---------------------------------------------------------------- public RemoteSession getRemoteSession(long sessionId) { - return remoteSessionCache.getSession(sessionId); + return remoteSessionCache.get(sessionId); } public List<Long> getRemoteSessions() { @@ -263,15 +266,15 @@ public class SessionRepository { } public void addRemoteSession(RemoteSession session) { - remoteSessionCache.putSession(session); + remoteSessionCache.put(session.getSessionId(), session); metrics.incAddedSessions(); } public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { int deleted = 0; for (long sessionId : getRemoteSessions()) { - RemoteSession session = remoteSessionCache.getSession(sessionId); - if (session == null) continue; // Internal sessions not in synch with zk, continue + RemoteSession session = remoteSessionCache.get(sessionId); + if (session == null) continue; // Internal sessions not in sync with zk, continue if (session.getStatus() == Session.Status.ACTIVATE) continue; if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { log.log(Level.FINE, () -> "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); @@ -324,14 +327,14 @@ public class SessionRepository { } private void checkForRemovedSessions(List<Long> sessions) { - for (RemoteSession session : remoteSessionCache.getSessions()) + for (RemoteSession session : remoteSessionCache.values()) if ( ! sessions.contains(session.getSessionId())) sessionRemoved(session.getSessionId()); } private void checkForAddedSessions(List<Long> sessions) { for (Long sessionId : sessions) - if (remoteSessionCache.getSession(sessionId) == null) + if (remoteSessionCache.get(sessionId) == null) sessionAdded(sessionId); } @@ -371,9 +374,15 @@ public class SessionRepository { public void delete(RemoteSession remoteSession) { LocalSession localSession = getLocalSession(remoteSession.getSessionId()); - if (localSession != null) - deleteLocalSession(localSession); remoteSession.deactivate(); + if (localSession == null) { + // This change will be picked up by directoryCache in this class, which will do the rest of the cleanup + try (Lock lock = lock(remoteSession.getSessionId())) { + remoteSession.delete(); + } + } else { + deleteLocalSession(localSession); + } } void prepare(RemoteSession session) { @@ -387,7 +396,7 @@ public class SessionRepository { private void sessionRemoved(long sessionId) { SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); if (watcher != null) watcher.close(); - remoteSessionCache.removeSession(sessionId); + remoteSessionCache.remove(sessionId); metrics.incRemovedSessions(); } @@ -405,7 +414,7 @@ public class SessionRepository { private void nodeChanged() { zkWatcherExecutor.execute(() -> { Multiset<Session.Status> sessionMetrics = HashMultiset.create(); - for (RemoteSession session : remoteSessionCache.getSessions()) { + for (RemoteSession session : remoteSessionCache.values()) { sessionMetrics.add(session.getStatus()); } metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); @@ -434,7 +443,7 @@ public class SessionRepository { private void synchronizeOnNew(List<Long> sessionList) { for (long sessionId : sessionList) { - RemoteSession session = remoteSessionCache.getSession(sessionId); + RemoteSession session = remoteSessionCache.get(sessionId); if (session == null) continue; // session might have been deleted after getting session list log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); session.confirmUpload(); @@ -623,7 +632,7 @@ public class SessionRepository { } catch (IllegalArgumentException e) { // We cannot be guaranteed that the file reference exists (it could be that it has not // been downloaded yet), and e.g when bootstrapping we cannot throw an exception in that case - log.log(Level.INFO, "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); + log.log(Level.FINE, "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); return; } ApplicationId applicationId = sessionZKClient.readApplicationId() diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index c5cf3db1ba0..2b3d3c3649a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -159,8 +159,9 @@ public class SessionZooKeeperClient { : Optional.of(ApplicationId.fromSerializedForm(idString)); } - void writeApplicationPackageReference(FileReference applicationPackageReference) { - configCurator.putData(applicationPackageReferencePath(), applicationPackageReference.value()); + void writeApplicationPackageReference(Optional<FileReference> applicationPackageReference) { + applicationPackageReference.ifPresent( + reference -> configCurator.putData(applicationPackageReferencePath(), reference.value())); } FileReference readApplicationPackageReference() { diff --git a/configserver/src/main/resources/mime.types b/configserver/src/main/resources/mime.types new file mode 100644 index 00000000000..f6672f7738d --- /dev/null +++ b/configserver/src/main/resources/mime.types @@ -0,0 +1,20 @@ +application/xml xml +application/java-archive jar +application/javascript js +application/json json + +text/html html +text/css css + +font/collection ttc +font/otf otf +font/ttf ttf +font/woff woff +font/woff2 woff2 + +image/x-icon ico +image/gif gif +image/jpeg jpg jpeg +image/png png +image/svg+xml svg +image/tiff tiff tif diff --git a/configserver/src/main/sh/stop-configserver b/configserver/src/main/sh/stop-configserver index 21dda50b0a2..ed415714592 100755 --- a/configserver/src/main/sh/stop-configserver +++ b/configserver/src/main/sh/stop-configserver @@ -95,11 +95,4 @@ if [ "$cloudconfig_server__multitenant" = "true" ] || [ "$VESPA_CONFIGSERVER_MUL vespa-run-as-vespa-user vespa-runserver -s logd -p $PIDFILE_LOGD -S fi -# Try shutting down this way in case of upgrade. Can be removed in later versions. vespa-run-as-vespa-user vespa-runserver -s configserver -p $PIDFILE_CONFIGSERVER -S - -if [ -e "$PIDFILE_CONFIGSERVER" ]; then - export UNPRIVILEGED=1 - export PID_FILE=$PIDFILE_CONFIGSERVER - exec vespa-run-as-vespa-user ${ROOT}/bin/jdisc_container_stop -fi diff --git a/configserver/src/test/apps/content/foo/bar/file-without-extension b/configserver/src/test/apps/content/foo/bar/file-without-extension new file mode 100644 index 00000000000..6b584e8ece5 --- /dev/null +++ b/configserver/src/test/apps/content/foo/bar/file-without-extension @@ -0,0 +1 @@ +content
\ No newline at end of file diff --git a/configserver/src/test/apps/content/foo/bar/test.txt b/configserver/src/test/apps/content/foo/bar/test.jar index 401a9f6e542..401a9f6e542 100644 --- a/configserver/src/test/apps/content/foo/bar/test.txt +++ b/configserver/src/test/apps/content/foo/bar/test.jar diff --git a/configserver/src/test/apps/content/foo/test1.txt b/configserver/src/test/apps/content/foo/test1.json index 5716ca5987c..5716ca5987c 100644 --- a/configserver/src/test/apps/content/foo/test1.txt +++ b/configserver/src/test/apps/content/foo/test1.json diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java index 20e52263350..d231c7ac60f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java @@ -1,6 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; +import com.google.common.base.Joiner; +import com.google.common.collect.Collections2; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.NOT_FOUND; import static com.yahoo.jdisc.Response.Status.OK; @@ -8,30 +18,21 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; - -import org.junit.Test; -import com.google.common.base.Joiner; -import com.google.common.collect.Collections2; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.jdisc.http.HttpRequest; - public abstract class ContentHandlerTestBase extends SessionHandlerTest { protected String baseUrl = "http://foo:1337/application/v2/tenant/default/session/1/content/"; @Test public void require_that_content_can_be_retrieved() throws IOException { assertContent("/test.txt", "foo\n"); - assertContent("/foo/", generateResultArray("foo/bar/", "foo/test1.txt", "foo/test2.txt")); - assertContent("/foo", generateResultArray("foo/")); - assertContent("/foo/test1.txt", "bar\n"); + assertContent("/foo/", generateResultArray("foo/bar/", "foo/test1.json", "foo/test2.txt"), "application/json"); + assertContent("/foo", generateResultArray("foo/"), "application/json"); + assertContent("/foo/test1.json", "bar\n", "application/json"); assertContent("/foo/test2.txt", "baz\n"); - assertContent("/foo/bar/", generateResultArray("foo/bar/test.txt")); - assertContent("/foo/bar", generateResultArray("foo/bar/")); - assertContent("/foo/bar/test.txt", "bim\n"); - assertContent("/foo/?recursive=true", generateResultArray("foo/bar/", "foo/bar/test.txt", "foo/test1.txt", "foo/test2.txt")); + assertContent("/foo/bar/", generateResultArray("foo/bar/file-without-extension", "foo/bar/test.jar"), "application/json"); + assertContent("/foo/bar", generateResultArray("foo/bar/"), "application/json"); + assertContent("/foo/bar/file-without-extension", "content"); + assertContent("/foo/bar/test.jar", "bim\n", "application/java-archive"); + assertContent("/foo/?recursive=true", generateResultArray("foo/bar/", "foo/bar/file-without-extension", "foo/bar/test.jar", "foo/test1.json", "foo/test2.txt"), "application/json"); } @Test @@ -58,21 +59,27 @@ public abstract class ContentHandlerTestBase extends SessionHandlerTest { "{\"status\":\"new\",\"md5\":\"d3b07384d113edec49eaa6238ad5ff00\",\"name\":\"" + baseUrl + "test.txt\"}"); assertStatus("/foo/?return=status", "[{\"status\":\"new\",\"md5\":\"\",\"name\":\"" + baseUrl + "foo/bar\"}," + - "{\"status\":\"new\",\"md5\":\"c157a79031e1c40f85931829bc5fc552\",\"name\":\"" + baseUrl + "foo/test1.txt\"}," + + "{\"status\":\"new\",\"md5\":\"c157a79031e1c40f85931829bc5fc552\",\"name\":\"" + baseUrl + "foo/test1.json\"}," + "{\"status\":\"new\",\"md5\":\"258622b1688250cb619f3c9ccaefb7eb\",\"name\":\"" + baseUrl + "foo/test2.txt\"}]"); assertStatus("/foo/?return=status&recursive=true", "[{\"status\":\"new\",\"md5\":\"\",\"name\":\"" + baseUrl + "foo/bar\"}," + - "{\"status\":\"new\",\"md5\":\"579cae6111b269c0129af36a2243b873\",\"name\":\"" + baseUrl + "foo/bar/test.txt\"}," + - "{\"status\":\"new\",\"md5\":\"c157a79031e1c40f85931829bc5fc552\",\"name\":\"" + baseUrl + "foo/test1.txt\"}," + + "{\"status\":\"new\",\"md5\":\"9a0364b9e99bb480dd25e1f0284c8555\",\"name\":\"" + baseUrl + "foo/bar/file-without-extension\"}," + + "{\"status\":\"new\",\"md5\":\"579cae6111b269c0129af36a2243b873\",\"name\":\"" + baseUrl + "foo/bar/test.jar\"}," + + "{\"status\":\"new\",\"md5\":\"c157a79031e1c40f85931829bc5fc552\",\"name\":\"" + baseUrl + "foo/test1.json\"}," + "{\"status\":\"new\",\"md5\":\"258622b1688250cb619f3c9ccaefb7eb\",\"name\":\"" + baseUrl + "foo/test2.txt\"}]"); } protected void assertContent(String path, String expectedContent) throws IOException { + assertContent(path, expectedContent, HttpResponse.DEFAULT_MIME_TYPE); + } + + protected void assertContent(String path, String expectedContent, String expectedContentType) throws IOException { HttpResponse response = doRequest(HttpRequest.Method.GET, path); assertNotNull(response); final String renderedString = SessionHandlerTest.getRenderedString(response); assertThat(renderedString, response.getStatus(), is(OK)); assertThat(renderedString, is(expectedContent)); + assertThat(response.getContentType(), is(expectedContentType)); } protected void assertStatus(String path, String expectedContent) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index cc1137ad9d8..ac1a5547646 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -136,7 +136,7 @@ public class SessionZooKeeperClientTest { public void require_that_application_package_file_reference_can_be_written_and_read() { final FileReference testRef = new FileReference("test-ref"); SessionZooKeeperClient zkc = createSessionZKClient(3); - zkc.writeApplicationPackageReference(testRef); + zkc.writeApplicationPackageReference(Optional.of(testRef)); assertThat(zkc.readApplicationPackageReference(), is(testRef)); } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 503bf2f2db1..cb177691fa3 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -9,17 +9,20 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.container.Container; +import com.yahoo.container.core.config.HandlersConfigurerDi; import com.yahoo.container.di.CloudSubscriberFactory; import com.yahoo.container.di.ComponentDeconstructor; -import com.yahoo.container.core.config.HandlersConfigurerDi; +import com.yahoo.container.handler.threadpool.ContainerThreadPool; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; +import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.osgi.MockOsgi; import java.io.File; import java.io.IOException; import java.util.LinkedHashSet; +import java.util.Map; import java.util.Random; import java.util.Set; @@ -135,6 +138,14 @@ public class HandlersConfigurerTestWrapper { protected void configure() { // Needed by e.g. SearchHandler bind(Linguistics.class).to(SimpleLinguistics.class).in(Scopes.SINGLETON); + bind(ContainerThreadPool.class).toInstance( + new ContainerThreadPool( + new ContainerThreadpoolConfig(new ContainerThreadpoolConfig.Builder()), + new Metric() { + @Override public void set(String key, Number val, Context ctx) {} + @Override public void add(String key, Number val, Context ctx) {} + @Override public Context createContext(Map<String, ?> properties) { return null;} + })); } }); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java index dd03d72d97d..a6042c541c0 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java @@ -27,8 +27,6 @@ public abstract class HttpResponse { private final Response parentResponse; - private Request.RequestType requestType; - /** * Creates a new HTTP response * @@ -126,12 +124,12 @@ public abstract class HttpResponse { } /** Sets the type classification of this request for metric collection purposes */ - public void setRequestType(Request.RequestType requestType) { this.requestType = requestType; } + public void setRequestType(Request.RequestType requestType) { parentResponse.setRequestType(requestType); } /** * Returns the type classification of this request for metric collection purposes, or null if not set. * When not set, the request type will be "read" for GET requests and "write" for other request methods. */ - public Request.RequestType getRequestType() { return requestType; } + public Request.RequestType getRequestType() { return parentResponse.getRequestType(); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java b/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java index d0cf07584e4..ecf120bfe44 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java @@ -4,8 +4,8 @@ package com.yahoo.container.jdisc; import com.yahoo.jdisc.application.MetricConsumer; /** - * <p>This is the interface to implement if one wishes to configure a non-default <code>MetricConsumer</code>. Simply - * add the implementing class as a component in your services.xml file.</p> + * This is the interface to implement if one wishes to configure a non-default <code>MetricConsumer</code>. Simply + * add the implementing class as a component in your services.xml file. * * @author Simon Thoresen Hult */ diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index ac302d6215b..d729e2371c9 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -78,7 +78,6 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler try { channel = new LazyContentChannel(httpRequest, responseHandler, metric, log); HttpResponse httpResponse = handle(httpRequest, channel); - request.setRequestType(httpResponse.getRequestType()); channel.setHttpResponse(httpResponse); // may or may not have already been done render(httpRequest, httpResponse, channel, jdiscRequest.creationTime(TimeUnit.MILLISECONDS)); } catch (Exception e) { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index e75c84376e0..226107c24b8 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -143,7 +143,6 @@ public final class ConfiguredApplication implements Application { @Override public void start() { qrConfig = getConfig(QrConfig.class); - slobrokRegistrator = registerInSlobrok(qrConfig); hackToInitializeServer(qrConfig); @@ -154,6 +153,7 @@ public final class ConfiguredApplication implements Application { portWatcher = new Thread(this::watchPortChange); portWatcher.setDaemon(true); portWatcher.start(); + slobrokRegistrator = registerInSlobrok(qrConfig); // marks this as up } /** diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index fb83f194cc5..26efb6c6312 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -4218,6 +4218,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.search.searchchain.ExecutionFactory, java.util.Optional)", diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index bb4df325762..bee25fbb47f 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -10,6 +10,7 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; +import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -19,38 +20,37 @@ import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.language.Linguistics; -import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; import com.yahoo.prelude.query.parser.ParseException; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.context.QueryContext; -import com.yahoo.search.query.ranking.SoftTimeout; -import com.yahoo.search.searchchain.ExecutionFactory; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.query.context.QueryContext; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.properties.DefaultProperties; +import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.ExecutionFactory; import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.statistics.ElapsedTime; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.SlimeUtils; import com.yahoo.statistics.Callback; import com.yahoo.statistics.Handle; import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Value; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; import java.io.IOException; @@ -62,6 +62,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -119,6 +120,16 @@ public class SearchHandler extends LoggingRequestHandler { @Inject public SearchHandler(Statistics statistics, Metric metric, + ContainerThreadPool threadpool, + AccessLog accessLog, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig config, + ExecutionFactory executionFactory) { + this(statistics, metric, threadpool.executor(), accessLog, queryProfileRegistry, config, executionFactory); + } + + public SearchHandler(Statistics statistics, + Metric metric, Executor executor, AccessLog accessLog, CompiledQueryProfileRegistry queryProfileRegistry, diff --git a/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java b/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java deleted file mode 100644 index 6e67386bfde..00000000000 --- a/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search; - -import com.yahoo.text.Utf8; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketException; -import java.util.Locale; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * As the name implies, a stupid, single-threaded bad-excuse-for-HTTP server. - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -public class StupidSingleThreadedHttpServer implements Runnable { - - private static final Logger log = Logger.getLogger(StupidSingleThreadedHttpServer.class.getName()); - - private final ServerSocket serverSocket; - private final int delaySeconds; - private Thread serverThread = null; - private CompletableFuture<String> requestFuture = new CompletableFuture<>(); - private final Pattern contentLengthPattern = Pattern.compile("content-length: (\\d+)", - Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE); - - public StupidSingleThreadedHttpServer() throws IOException { - this(0, 0); - } - - public StupidSingleThreadedHttpServer(int port, int delaySeconds) throws IOException { - this.delaySeconds = delaySeconds; - this.serverSocket = new ServerSocket(port); - } - - public void start() { - serverThread = new Thread(this); - serverThread.setDaemon(true); - serverThread.start(); - } - - public void run() { - try { - while(true) { - Socket socket = serverSocket.accept(); - StringBuilder request = new StringBuilder(); - socket.setSoLinger(true, 60); - BufferedReader in = new BufferedReader( - new InputStreamReader( - socket.getInputStream())); - - int contentLength = -1; - String inputLine; - while (!"".equals(inputLine = in.readLine())) { //read header: - request.append(inputLine).append("\r\n"); - if (inputLine.toLowerCase(Locale.US).contains("content-length")) { - Matcher contentLengthMatcher = contentLengthPattern.matcher(inputLine); - if (contentLengthMatcher.matches()) { - contentLength = Integer.parseInt(contentLengthMatcher.group(1)); - } - } - } - request.append("\r\n"); - - if (contentLength < 0) { - System.err.println("WARNING! Got no Content-Length header!!"); - } else { - char[] requestBody = new char[contentLength]; - int readRemaining = contentLength; - - do { - int read = in.read(requestBody, (contentLength - readRemaining), readRemaining); - if (read < 0) { - throw new IllegalStateException("Should not get EOF here!!"); - } - readRemaining -= read; - } while (readRemaining > 0); - - request.append(new String(requestBody)); - } - - // Simulate service slowness - if (delaySeconds > 0) { - try { - System.out.println(this.getClass().getCanonicalName() + " sleeping in " + delaySeconds + " s before responding..."); - Thread.sleep((long) (delaySeconds * 1000)); - System.out.println("done sleeping, responding"); - } catch (InterruptedException e) { - //ignore - } - } - - socket.getOutputStream().write(getResponse(request.toString())); - socket.getOutputStream().flush(); - in.close(); - socket.close(); - - boolean wasCompleted = requestFuture.complete(request.toString()); - if (!wasCompleted) { - log.log(Level.INFO, "Only the first request will be stored, ignoring. " - + "Old value: " + requestFuture.get() - + ", New value: " + request.toString()); - } - } - } catch (SocketException se) { - if ("Socket closed".equals(se.getMessage())) { - //ignore - } else { - throw new RuntimeException(se); - } - } catch (IOException|InterruptedException|ExecutionException e) { - throw new RuntimeException(e); - } - } - - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 0\r\n" + - "\r\n"); - } - - protected byte[] getResponseBody() { - return new byte[0]; - } - - public void stop() { - if (!serverSocket.isClosed()) { - try { - serverSocket.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - try { - serverThread.interrupt(); - } catch (Exception e) { - //ignore - } - } - - public int getServerPort() { - return serverSocket.getLocalPort(); - } - - public String getRequest() { - try { - return requestFuture.get(1, TimeUnit.MINUTES); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - throw new AssertionError("Failed waiting for request. ", e); - } - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 317b29cf621..23a9ca79b53 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartF import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.InputStream; +import java.net.URI; import java.util.List; import java.util.Map; import java.util.Optional; @@ -55,6 +56,16 @@ public interface ConfigServer { */ InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters); + /** + * Gets the contents of a file inside the current application package for a given deployment. If the path is to + * a directly, a JSON list with URLs to contents is returned. + * + * @param deployment deployment to get application package content for + * @param path path within package to get + * @param requestUri request URI on the controller, used to rewrite paths in response from config server + */ + ProxyResponse getApplicationPackageContent(DeploymentId deployment, String path, URI requestUri); + List<ClusterMetrics> getDeploymentMetrics(DeploymentId deployment); List<ProtonMetrics> getProtonMetrics(DeploymentId deployment); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ProxyResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ProxyResponse.java new file mode 100644 index 00000000000..8570155861a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ProxyResponse.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * @author valerijf + */ +public class ProxyResponse extends HttpResponse { + + private final String content; + private final String contentType; + + public ProxyResponse(String content, String contentType, int status) { + super(status); + this.content = content; + this.contentType = contentType; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(content.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { + return Optional.ofNullable(contentType).orElseGet(super::getContentType); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 571fd649f04..95f394aefef 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -111,6 +111,7 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/job/{*}", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/nodes", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/clusters", + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/content/{*}", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/logs", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/suspended", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/service/{*}", diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7da8ac74936..81b4c4cc72f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -92,6 +92,9 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.Exceptions; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; import javax.ws.rs.ForbiddenException; import javax.ws.rs.InternalServerErrorException; @@ -101,6 +104,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.Paths; import java.security.DigestInputStream; import java.security.Principal; import java.security.PublicKey; @@ -120,9 +124,6 @@ import java.util.Scanner; import java.util.StringJoiner; import java.util.logging.Level; import java.util.stream.Collectors; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; @@ -232,6 +233,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/content/{*}")) return content(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/metrics")) return metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("endpointId"))); @@ -1316,6 +1318,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return response; } + private HttpResponse content(String tenantName, String applicationName, String instanceName, String environment, String region, String restPath, HttpRequest request) { + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); + return controller.serviceRegistry().configServer().getApplicationPackageContent(deploymentId, "/" + restPath, request.getUri()); + } + private HttpResponse updateTenant(String tenantName, HttpRequest request) { getTenantOrThrow(tenantName); TenantName tenant = TenantName.from(tenantName); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 1dd3b4a7a47..85e456afa60 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ProxyResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; @@ -42,6 +43,7 @@ import com.yahoo.vespa.serviceview.bindings.ServiceView; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; @@ -516,6 +518,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)); } + @Override + public ProxyResponse getApplicationPackageContent(DeploymentId deployment, String path, URI requestUri) { + return new ProxyResponse("{\"path\":\"" + path + "\"}", "application/json", 200); + } + public void setLogStream(String log) { this.log = log; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 2f514bd05cd..8a16e066119 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -495,6 +495,14 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "INFO - All good"); + // Get content - root + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-central-1/content/", GET).userIdentity(USER_ID), + "{\"path\":\"/\"}"); + // Get content - ignore query params + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/instance/default/environment/dev/region/us-central-1/content/bar/file.json?query=param", GET).userIdentity(USER_ID), + "{\"path\":\"/bar/file.json\"}"); + + updateMetrics(); // GET metrics diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 12201af56bb..0e5e43cc989 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -104,6 +104,8 @@ endfunction() function(setup_vespa_default_build_settings_debian_10) message("-- Setting up default build settings for debian 10") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) endfunction() diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index 472440d7863..166c3be5e66 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -51,11 +51,11 @@ namespace { string StoragePolicy::init() { string error = ExternSlobrokPolicy::init(); - if (error.length() > 0) { + if (!error.empty()) { return error; } - if (!_clusterConfigId.length()) { + if (_clusterConfigId.empty()) { _clusterConfigId = createConfigId(_clusterName); } @@ -72,9 +72,7 @@ string StoragePolicy::init() return ""; } -StoragePolicy::~StoragePolicy() -{ -} +StoragePolicy::~StoragePolicy() = default; string StoragePolicy::createConfigId(const string & clusterName) const @@ -102,7 +100,7 @@ void StoragePolicy::configure(std::unique_ptr<vespa::config::content::StorDistributionConfig> config) { try { - _nextDistribution.reset(new storage::lib::Distribution(*config)); + _nextDistribution = std::make_unique<storage::lib::Distribution>(*config); } catch (const std::exception& e) { LOG(warning, "Got exception when configuring distribution, config id was %s", _clusterConfigId.c_str()); throw e; @@ -167,25 +165,23 @@ StoragePolicy::doSelect(mbus::RoutingContext &context) // Pick a distributor using ideal state algorithm try { - // Update distribution here, to make it not take lock in average case - if (_nextDistribution.get() != 0) { + // Update distribution here, to make it not take lock in average case + if (_nextDistribution) { _distribution = std::move(_nextDistribution); _nextDistribution.reset(); } assert(_distribution.get()); distributor = _distribution->getIdealDistributorNode(*_state, id); } catch (storage::lib::TooFewBucketBitsInUseException& e) { - mbus::Reply::UP reply( - new WrongDistributionReply(_state->toString())); + auto reply = std::make_unique<WrongDistributionReply>(_state->toString()); reply->addError(mbus::Error( DocumentProtocol::ERROR_WRONG_DISTRIBUTION, "Too few distribution bits used for given cluster state")); context.setReply(std::move(reply)); return; - } catch (storage::lib::NoDistributorsAvailableException& e) { - // No distributors available in current cluster state. Remove - // cluster state we cannot use and send to random target + // No distributors available in current cluster state. Remove + // cluster state we cannot use and send to random target _state.reset(); distributor = -1; } @@ -240,8 +236,8 @@ StoragePolicy::updateStateFromReply(WrongDistributionReply& wdr) { std::unique_ptr<storage::lib::ClusterState> newState( new storage::lib::ClusterState(wdr.getSystemState())); - if (_state.get() == 0 || newState->getVersion() >= _state->getVersion()) { - if (_state.get()) { + if (!_state || newState->getVersion() >= _state->getVersion()) { + if (_state) { wdr.getTrace().trace(1, make_string("System state changed from version %u to %u", _state->getVersion(), newState->getVersion())); } else { diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index fe9d9985c6a..76c1a63b881 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -24,6 +24,7 @@ vespa_define_module( src/tests/eval/node_types src/tests/eval/param_usage src/tests/eval/simple_tensor + src/tests/eval/simple_value src/tests/eval/tensor_function src/tests/eval/tensor_lambda src/tests/eval/tensor_spec diff --git a/eval/src/tests/eval/simple_value/CMakeLists.txt b/eval/src/tests/eval/simple_value/CMakeLists.txt new file mode 100644 index 00000000000..429d3ffaf3d --- /dev/null +++ b/eval/src/tests/eval/simple_value/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_simple_value_test_app TEST + SOURCES + simple_value_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_simple_value_test_app COMMAND eval_simple_value_test_app) diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp new file mode 100644 index 00000000000..32a099afce3 --- /dev/null +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -0,0 +1,167 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +std::vector<Layout> layouts = { + {}, + {x(3)}, + {x(3),y(5)}, + {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), + {x({"a","b","c"})}, + {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +}; + +std::vector<Layout> join_layouts = { + {}, {}, + {x(5)}, {x(5)}, + {x(5)}, {y(5)}, + {x(5)}, {x(5),y(5)}, + {y(3)}, {x(2),z(3)}, + {x(3),y(5)}, {y(5),z(7)}, + float_cells({x(3),y(5)}), {y(5),z(7)}, + {x(3),y(5)}, float_cells({y(5),z(7)}), + float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {y({"foo","bar","baz"})}, + {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, + {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), + float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) +}; + +TensorSpec simple_tensor_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + Stash stash; + const auto &engine = SimpleTensorEngine::ref(); + auto lhs = engine.from_spec(a); + auto rhs = engine.from_spec(b); + const auto &result = engine.join(*lhs, *rhs, function, stash); + return engine.to_spec(result); +} + +TensorSpec simple_value_new_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { + auto lhs = new_value_from_spec(a, SimpleValueBuilderFactory()); + auto rhs = new_value_from_spec(b, SimpleValueBuilderFactory()); + auto result = new_join(*lhs, *rhs, function, SimpleValueBuilderFactory()); + return spec_from_new_value(*result); +} + +TEST(SimpleValueTest, simple_values_can_be_converted_from_and_to_tensor_spec) { + for (const auto &layout: layouts) { + TensorSpec expect = spec(layout, N()); + std::unique_ptr<NewValue> value = new_value_from_spec(expect, SimpleValueBuilderFactory()); + TensorSpec actual = spec_from_new_value(*value); + EXPECT_EQ(actual, expect); + } +} + +TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { + ValueType type = ValueType::from_spec("tensor<float>(x{},y[2],z{})"); + SimpleValueBuilderFactory factory; + std::unique_ptr<ValueBuilder<float>> builder = factory.create_value_builder<float>(type); + float seq = 0.0; + for (vespalib::string x: {"a", "b", "c"}) { + for (vespalib::string y: {"aa", "bb"}) { + auto subspace = builder->add_subspace({x, y}); + EXPECT_EQ(subspace.size(), 2); + subspace[0] = seq + 1.0; + subspace[1] = seq + 5.0; + seq += 10.0; + } + seq += 100.0; + } + std::unique_ptr<NewValue> value = builder->build(std::move(builder)); + EXPECT_EQ(value->index().size(), 6); + auto view = value->index().create_view({0}); + vespalib::stringref query = "b"; + vespalib::stringref label; + size_t subspace; + view->lookup({&query}); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "aa"); + EXPECT_EQ(subspace, 2); + EXPECT_TRUE(view->next_result({&label}, subspace)); + EXPECT_EQ(label, "bb"); + EXPECT_EQ(subspace, 3); + EXPECT_FALSE(view->next_result({&label}, subspace)); +} + +TEST(SimpleValueTest, dense_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(a{},b[6],c[5],d[4],h{})"); + auto plan = DenseJoinPlan(lhs, rhs); + std::vector<size_t> expect_loop = {30,4,6}; + std::vector<size_t> expect_lhs_stride = {6,0,1}; + std::vector<size_t> expect_rhs_stride = {4,1,0}; + EXPECT_EQ(plan.lhs_size, 180); + EXPECT_EQ(plan.rhs_size, 120); + EXPECT_EQ(plan.out_size, 720); + EXPECT_EQ(plan.loop_cnt, expect_loop); + EXPECT_EQ(plan.lhs_stride, expect_lhs_stride); + EXPECT_EQ(plan.rhs_stride, expect_rhs_stride); +} + +TEST(SimpleValueTest, sparse_join_plan_can_be_created) { + auto lhs = ValueType::from_spec("tensor(a{},b[6],c[5],e[3],f[2],g{})"); + auto rhs = ValueType::from_spec("tensor(b[6],c[5],d[4],g{},h{})"); + auto plan = SparseJoinPlan(lhs, rhs); + using SRC = SparseJoinPlan::Source; + std::vector<SRC> expect_sources = {SRC::LHS,SRC::BOTH,SRC::RHS}; + std::vector<size_t> expect_lhs_overlap = {1}; + std::vector<size_t> expect_rhs_overlap = {0}; + EXPECT_EQ(plan.sources, expect_sources); + EXPECT_EQ(plan.lhs_overlap, expect_lhs_overlap); + EXPECT_EQ(plan.rhs_overlap, expect_rhs_overlap); +} + +TEST(SimpleValueTest, dense_join_plan_can_be_executed) { + auto plan = DenseJoinPlan(ValueType::from_spec("tensor(a[2])"), + ValueType::from_spec("tensor(b[3])")); + std::vector<int> a({1, 2}); + std::vector<int> b({3, 4, 5}); + std::vector<int> c(6, 0); + std::vector<int> expect = {3,4,5,6,8,10}; + ASSERT_EQ(plan.out_size, 6); + int *dst = &c[0]; + auto cell_join = [&](size_t a_idx, size_t b_idx) { *dst++ = (a[a_idx] * b[b_idx]); }; + plan.execute(0, 0, cell_join); + EXPECT_EQ(c, expect); +} + +TEST(SimpleValueTest, new_generic_join_works_for_simple_values) { + ASSERT_TRUE((join_layouts.size() % 2) == 0); + for (size_t i = 0; i < join_layouts.size(); i += 2) { + TensorSpec lhs = spec(join_layouts[i], Div16(N())); + TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = simple_tensor_join(lhs, rhs, fun); + auto actual = simple_value_new_join(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 54bcd0694e2..2b103a91b81 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -266,18 +266,28 @@ TEST("require that dimension names can be obtained") { EXPECT_EQUAL(type("tensor<float>(y[10],x[30],z{})").dimension_names(), str_list({"x", "y", "z"})); } -TEST("require that nontrivial dimensions can be obtained") { +TEST("require that nontrivial indexed dimensions can be obtained") { auto my_check = [](const auto &list) { - ASSERT_EQUAL(list.size(), 2u); + ASSERT_EQUAL(list.size(), 1u); EXPECT_EQUAL(list[0].name, "x"); EXPECT_EQUAL(list[0].size, 10u); - EXPECT_EQUAL(list[1].name, "y"); - EXPECT_TRUE(list[1].is_mapped()); }; - EXPECT_TRUE(type("double").nontrivial_dimensions().empty()); - TEST_DO(my_check(type("tensor(x[10],y{})").nontrivial_dimensions())); - TEST_DO(my_check(type("tensor(a[1],b[1],x[10],y{},z[1])").nontrivial_dimensions())); + EXPECT_TRUE(type("double").nontrivial_indexed_dimensions().empty()); + TEST_DO(my_check(type("tensor(x[10],y{})").nontrivial_indexed_dimensions())); + TEST_DO(my_check(type("tensor(a[1],b[1],x[10],y{},z[1])").nontrivial_indexed_dimensions())); +} + +TEST("require that mapped dimensions can be obtained") { + auto my_check = [](const auto &list) + { + ASSERT_EQUAL(list.size(), 1u); + EXPECT_EQUAL(list[0].name, "x"); + EXPECT_TRUE(list[0].is_mapped()); + }; + EXPECT_TRUE(type("double").mapped_dimensions().empty()); + TEST_DO(my_check(type("tensor(x{},y[10])").mapped_dimensions())); + TEST_DO(my_check(type("tensor(a[1],b[1],x{},y[10],z[1])").mapped_dimensions())); } TEST("require that dimension index can be obtained") { @@ -315,6 +325,14 @@ TEST("require that type-related predicate functions work as expected") { TEST_DO(verify_predicates(type("tensor<float>(x[5],y{})"), false, false, true, false, false)); } +TEST("require that mapped dimensions can be counted") { + EXPECT_EQUAL(type("double").count_mapped_dimensions(), 0u); + EXPECT_EQUAL(type("tensor(x[5],y[5])").count_mapped_dimensions(), 0u); + EXPECT_EQUAL(type("tensor(x{},y[5])").count_mapped_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x[5],y{})").count_mapped_dimensions(), 1u); + EXPECT_EQUAL(type("tensor(x{},y{})").count_mapped_dimensions(), 2u); +} + TEST("require that dense subspace size calculation works as expected") { EXPECT_EQUAL(type("error").dense_subspace_size(), 1u); EXPECT_EQUAL(type("double").dense_subspace_size(), 1u); diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 002a027c3a9..973245607de 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_add_library(eval_eval OBJECT param_usage.cpp simple_tensor.cpp simple_tensor_engine.cpp + simple_value.cpp string_stuff.cpp tensor.cpp tensor_engine.cpp diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp new file mode 100644 index 00000000000..e8ab26078e6 --- /dev/null +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -0,0 +1,413 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "simple_value.h" +#include "tensor_spec.h" +#include "inline_operation.h" +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <vespa/vespalib/util/overload.h> + +#include <vespa/log/log.h> +LOG_SETUP(".eval.simple_value"); + +namespace vespalib::eval { + +//----------------------------------------------------------------------------- + +namespace { + +struct CreateSimpleValueBuilderBase { + template <typename T> static std::unique_ptr<ValueBuilderBase> invoke(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in) + { + assert(check_cell_type<T>(type.cell_type())); + return std::make_unique<SimpleValueT<T>>(type, num_mapped_dims_in, subspace_size_in); + } +}; + +struct CreateValueFromTensorSpec { + template <typename T> static std::unique_ptr<NewValue> invoke(const ValueType &type, const TensorSpec &spec, const ValueBuilderFactory &factory) { + using SparseKey = std::vector<vespalib::stringref>; + using DenseMap = std::map<size_t,T>; + std::map<SparseKey,DenseMap> map; + for (const auto &entry: spec.cells()) { + SparseKey sparse_key; + size_t dense_key = 0; + for (const auto &dim: type.dimensions()) { + auto pos = entry.first.find(dim.name); + assert(pos != entry.first.end()); + assert(pos->second.is_mapped() == dim.is_mapped()); + if (dim.is_mapped()) { + sparse_key.emplace_back(pos->second.name); + } else { + dense_key = (dense_key * dim.size) + pos->second.index; + } + } + map[sparse_key][dense_key] = entry.second; + } + auto builder = factory.create_value_builder<T>(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size()); + for (const auto &entry: map) { + auto subspace = builder->add_subspace(entry.first); + for (const auto &cell: entry.second) { + subspace[cell.first] = cell.second; + } + } + return builder->build(std::move(builder)); + } +}; + +struct CreateTensorSpecFromValue { + template <typename T> static TensorSpec invoke(const NewValue &value) { + auto cells = value.cells().typify<T>(); + TensorSpec spec(value.type().to_spec()); + size_t subspace_id = 0; + size_t subspace_size = value.type().dense_subspace_size(); + std::vector<vespalib::stringref> labels(value.type().count_mapped_dimensions()); + std::vector<vespalib::stringref*> label_refs; + for (auto &label: labels) { + label_refs.push_back(&label); + } + auto view = value.index().create_view({}); + view->lookup({}); + while (view->next_result(label_refs, subspace_id)) { + size_t label_idx = 0; + TensorSpec::Address addr; + for (const auto &dim: value.type().dimensions()) { + if (dim.is_mapped()) { + addr.emplace(dim.name, labels[label_idx++]); + } + } + for (size_t i = 0; i < subspace_size; ++i) { + size_t dense_key = i; + for (auto dim = value.type().dimensions().rbegin(); + dim != value.type().dimensions().rend(); ++dim) + { + if (dim->is_indexed()) { + size_t label = dense_key % dim->size; + addr.emplace(dim->name, label).first->second = TensorSpec::Label(label); + dense_key /= dim->size; + } + } + spec.add(addr, cells[(subspace_size * subspace_id) + i]); + } + } + return spec; + } +}; + +class SimpleValueView : public NewValue::Index::View { +private: + using Addr = std::vector<vespalib::string>; + using Map = std::map<Addr,size_t>; + using Itr = Map::const_iterator; + + const Map &_index; + size_t _num_mapped_dims; + std::vector<size_t> _match_dims; + std::vector<size_t> _extract_dims; + Addr _query; + Itr _pos; + + bool is_direct_lookup() const { return (_match_dims.size() == _num_mapped_dims); } + bool is_match() const { + assert(_pos->first.size() == _num_mapped_dims); + for (size_t idx: _match_dims) { + if (_query[idx] != _pos->first[idx]) { + return false; + } + } + return true; + } + +public: + SimpleValueView(const Map &index, const std::vector<size_t> &match_dims, size_t num_mapped_dims) + : _index(index), _num_mapped_dims(num_mapped_dims), _match_dims(match_dims), _extract_dims(), _query(num_mapped_dims, ""), _pos(_index.end()) + { + auto pos = _match_dims.begin(); + for (size_t i = 0; i < _num_mapped_dims; ++i) { + if ((pos == _match_dims.end()) || (*pos != i)) { + _extract_dims.push_back(i); + } else { + ++pos; + } + } + assert(pos == _match_dims.end()); + assert((_match_dims.size() + _extract_dims.size()) == _num_mapped_dims); + } + + void lookup(const std::vector<const vespalib::stringref*> &addr) override { + assert(addr.size() == _match_dims.size()); + for (size_t i = 0; i < _match_dims.size(); ++i) { + _query[_match_dims[i]] = *addr[i]; + } + if (is_direct_lookup()) { + _pos = _index.find(_query); + } else { + _pos = _index.begin(); + } + } + + bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) override { + assert(addr_out.size() == _extract_dims.size()); + while (_pos != _index.end()) { + if (is_match()) { + for (size_t i = 0; i < _extract_dims.size(); ++i) { + *addr_out[i] = _pos->first[_extract_dims[i]]; + } + idx_out = _pos->second; + if (is_direct_lookup()) { + _pos = _index.end(); + } else { + ++_pos; + } + return true; + } + ++_pos; + } + return false; + } +}; + +// Contains various state needed to perform the sparse part (all +// mapped dimensions) of the join operation. Performs swapping of +// sparse indexes to ensure that we look up entries from the smallest +// index in the largest index. +struct SparseJoinState { + bool swapped; + const NewValue::Index &first_index; + const NewValue::Index &second_index; + const std::vector<size_t> &second_view_dims; + std::vector<vespalib::stringref> full_address; + std::vector<vespalib::stringref*> first_address; + std::vector<const vespalib::stringref*> address_overlap; + std::vector<vespalib::stringref*> second_only_address; + size_t lhs_subspace; + size_t rhs_subspace; + size_t &first_subspace; + size_t &second_subspace; + + SparseJoinState(const SparseJoinPlan &plan, const NewValue::Index &lhs, const NewValue::Index &rhs) + : swapped(rhs.size() < lhs.size()), + first_index(swapped ? rhs : lhs), second_index(swapped ? lhs : rhs), + second_view_dims(swapped ? plan.lhs_overlap : plan.rhs_overlap), + full_address(plan.sources.size()), + first_address(), address_overlap(), second_only_address(), + lhs_subspace(), rhs_subspace(), + first_subspace(swapped ? rhs_subspace : lhs_subspace), + second_subspace(swapped ? lhs_subspace : rhs_subspace) + { + auto first_source = swapped ? SparseJoinPlan::Source::RHS : SparseJoinPlan::Source::LHS; + for (size_t i = 0; i < full_address.size(); ++i) { + if (plan.sources[i] == SparseJoinPlan::Source::BOTH) { + first_address.push_back(&full_address[i]); + address_overlap.push_back(&full_address[i]); + } else if (plan.sources[i] == first_source) { + first_address.push_back(&full_address[i]); + } else { + second_only_address.push_back(&full_address[i]); + } + } + } + ~SparseJoinState(); +}; +SparseJoinState::~SparseJoinState() = default; + +// Treats all values as mixed tensors. Needs output cell type as well +// as input cell types since output cell type cannot always be +// directly inferred. +struct GenericJoin { + template <typename LCT, typename RCT, typename OCT, typename Fun> static std::unique_ptr<NewValue> + invoke(const NewValue &lhs, const NewValue &rhs, join_fun_t function, + const SparseJoinPlan &sparse_plan, const DenseJoinPlan &dense_plan, + const ValueType &res_type, const ValueBuilderFactory &factory) + { + Fun fun(function); + auto lhs_cells = lhs.cells().typify<LCT>(); + auto rhs_cells = rhs.cells().typify<RCT>(); + SparseJoinState state(sparse_plan, lhs.index(), rhs.index()); + auto builder = factory.create_value_builder<OCT>(res_type, sparse_plan.sources.size(), dense_plan.out_size, state.first_index.size()); + auto outer = state.first_index.create_view({}); + auto inner = state.second_index.create_view(state.second_view_dims); + outer->lookup({}); + while (outer->next_result(state.first_address, state.first_subspace)) { + inner->lookup(state.address_overlap); + while (inner->next_result(state.second_only_address, state.second_subspace)) { + OCT *dst = builder->add_subspace(state.full_address).begin(); + auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; + dense_plan.execute(dense_plan.lhs_size * state.lhs_subspace, dense_plan.rhs_size * state.rhs_subspace, join_cells); + } + } + return builder->build(std::move(builder)); + } +}; + +} // namespace <unnamed> + +//----------------------------------------------------------------------------- + +void +SimpleValue::add_mapping(const std::vector<vespalib::stringref> &addr) +{ + size_t id = _index.size(); + std::vector<vespalib::string> my_addr; + for (const auto &label: addr) { + my_addr.push_back(label); + } + auto res = _index.emplace(std::move(my_addr), id); + assert(res.second); +} + +SimpleValue::SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in) + : _type(type), + _num_mapped_dims(num_mapped_dims_in), + _subspace_size(subspace_size_in), + _index() +{ + assert(_type.count_mapped_dimensions() == _num_mapped_dims); + assert(_type.dense_subspace_size() == _subspace_size); +} + +SimpleValue::~SimpleValue() = default; + +std::unique_ptr<NewValue::Index::View> +SimpleValue::create_view(const std::vector<size_t> &dims) const +{ + return std::make_unique<SimpleValueView>(_index, dims, _num_mapped_dims); +} + +//----------------------------------------------------------------------------- + +template <typename T> +SimpleValueT<T>::SimpleValueT(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in) + : SimpleValue(type, num_mapped_dims_in, subspace_size_in), + _cells() +{ +} + +template <typename T> +SimpleValueT<T>::~SimpleValueT() = default; + +template <typename T> +ArrayRef<T> +SimpleValueT<T>::add_subspace(const std::vector<vespalib::stringref> &addr) +{ + size_t old_size = _cells.size(); + assert(old_size == (index().size() * subspace_size())); + add_mapping(addr); + _cells.resize(old_size + subspace_size()); + return ArrayRef<T>(&_cells[old_size], subspace_size()); +} + +//----------------------------------------------------------------------------- + +std::unique_ptr<ValueBuilderBase> +SimpleValueBuilderFactory::create_value_builder_base(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t) const +{ + return typify_invoke<1,TypifyCellType,CreateSimpleValueBuilderBase>(type.cell_type(), type, num_mapped_dims_in, subspace_size_in); +} + +//----------------------------------------------------------------------------- + +DenseJoinPlan::DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : lhs_size(1), rhs_size(1), out_size(1), loop_cnt(), lhs_stride(), rhs_stride() +{ + enum class Case { NONE, LHS, RHS, BOTH }; + Case prev_case = Case::NONE; + auto update_plan = [&](Case my_case, size_t my_size, size_t in_lhs, size_t in_rhs) { + if (my_case == prev_case) { + assert(!loop_cnt.empty()); + loop_cnt.back() *= my_size; + } else { + loop_cnt.push_back(my_size); + lhs_stride.push_back(in_lhs); + rhs_stride.push_back(in_rhs); + prev_case = my_case; + } + }; + auto visitor = overload + { + [&](visit_ranges_first, const auto &a) { update_plan(Case::LHS, a.size, 1, 0); }, + [&](visit_ranges_second, const auto &b) { update_plan(Case::RHS, b.size, 0, 1); }, + [&](visit_ranges_both, const auto &a, const auto &) { update_plan(Case::BOTH, a.size, 1, 1); } + }; + auto lhs_dims = lhs_type.nontrivial_indexed_dimensions(); + auto rhs_dims = rhs_type.nontrivial_indexed_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); + for (size_t i = loop_cnt.size(); i-- > 0; ) { + out_size *= loop_cnt[i]; + if (lhs_stride[i] != 0) { + lhs_stride[i] = lhs_size; + lhs_size *= loop_cnt[i]; + } + if (rhs_stride[i] != 0) { + rhs_stride[i] = rhs_size; + rhs_size *= loop_cnt[i]; + } + } +} + +DenseJoinPlan::~DenseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type) + : sources(), lhs_overlap(), rhs_overlap() +{ + size_t lhs_idx = 0; + size_t rhs_idx = 0; + auto visitor = overload + { + [&](visit_ranges_first, const auto &) { + sources.push_back(Source::LHS); + ++lhs_idx; + }, + [&](visit_ranges_second, const auto &) { + sources.push_back(Source::RHS); + ++rhs_idx; + }, + [&](visit_ranges_both, const auto &, const auto &) { + sources.push_back(Source::BOTH); + lhs_overlap.push_back(lhs_idx++); + rhs_overlap.push_back(rhs_idx++); + } + }; + auto lhs_dims = lhs_type.mapped_dimensions(); + auto rhs_dims = rhs_type.mapped_dimensions(); + visit_ranges(visitor, lhs_dims.begin(), lhs_dims.end(), rhs_dims.begin(), rhs_dims.end(), + [](const auto &a, const auto &b){ return (a.name < b.name); }); +} + +SparseJoinPlan::~SparseJoinPlan() = default; + +//----------------------------------------------------------------------------- + +using JoinTypify = TypifyValue<TypifyCellType,operation::TypifyOp2>; + +std::unique_ptr<NewValue> new_join(const NewValue &a, const NewValue &b, join_fun_t function, const ValueBuilderFactory &factory) { + auto res_type = ValueType::join(a.type(), b.type()); + assert(!res_type.is_error()); + SparseJoinPlan sparse_plan(a.type(), b.type()); + DenseJoinPlan dense_plan(a.type(), b.type()); + return typify_invoke<4,JoinTypify,GenericJoin>(a.type().cell_type(), b.type().cell_type(), res_type.cell_type(), function, + a, b, function, sparse_plan, dense_plan, res_type, factory); +} + +//----------------------------------------------------------------------------- + +std::unique_ptr<NewValue> new_value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { + ValueType type = ValueType::from_spec(spec.type()); + assert(!type.is_error()); + return typify_invoke<1,TypifyCellType,CreateValueFromTensorSpec>(type.cell_type(), type, spec, factory); +} + +//----------------------------------------------------------------------------- + +TensorSpec spec_from_new_value(const NewValue &value) { + return typify_invoke<1,TypifyCellType,CreateTensorSpecFromValue>(value.type().cell_type(), value); +} + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/simple_value.h b/eval/src/vespa/eval/eval/simple_value.h new file mode 100644 index 00000000000..892dd6f1da6 --- /dev/null +++ b/eval/src/vespa/eval/eval/simple_value.h @@ -0,0 +1,275 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" +#include "value_type.h" +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/vespalib/stllike/string.h> +#include <vector> +#include <map> + +namespace vespalib { class Stash; } + +namespace vespalib::eval { + +class TensorSpec; + +using TypedCells = ::vespalib::tensor::TypedCells; + +/** + * Experimental interface layer that will be moved into Value when all + * existing implementations are able to implement it. This interface + * will try to unify scalars, dense tensors, sparse tensors and mixed + * tensors while also enabling operations to be implemented + * efficiently using this interface without having knowledge about the + * actual implementation. Baseline operations will treat all values as + * mixed tensors. Simplified and optimized variants may replace them + * as done today based on type knowledge. + * + * All values are expected to be separated into a continuous area + * storing cells as concatenated dense subspaces, and an index + * structure used to look up label combinations; mapping them into a + * set of dense subspaces. + **/ +struct NewValue : Value { + + // Root lookup structure for mapping labels to dense subspace indexes + struct Index { + + // A view able to look up dense subspace indexes from labels + // specifying a partial address for the dimensions given to + // create_view. A view is re-usable. Lookups are performed by + // calling the lookup function and lookup results are + // extracted using the next_result function. + struct View { + + // look up dense subspace indexes from labels specifying a + // partial address for the dimensions given to + // create_view. Results from the lookup is extracted using + // the next_result function. + virtual void lookup(const std::vector<const vespalib::stringref*> &addr) = 0; + + // Extract the next result (if any) from the previous + // lookup into the given partial address and index. Only + // the labels for the dimensions NOT specified in + // create_view will be extracted here. + virtual bool next_result(const std::vector<vespalib::stringref*> &addr_out, size_t &idx_out) = 0; + + virtual ~View() {} + }; + + // total number of mappings (equal to the number of dense subspaces) + virtual size_t size() const = 0; + + // create a view able to look up dense subspaces based on + // labels from a subset of the mapped dimensions. + virtual std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const = 0; + + virtual ~Index() {} + }; + virtual TypedCells cells() const = 0; + virtual const Index &index() const = 0; + virtual ~NewValue() {} +}; + +/** + * Tagging interface used as return type from factories before + * downcasting to actual builder with specialized cell type. + **/ +struct ValueBuilderBase { + virtual ~ValueBuilderBase() {} +}; + +/** + * Interface used to build a value one dense subspace at a + * time. Enables decoupling of what the value should contain from how + * to store the value. + **/ +template <typename T> +struct ValueBuilder : ValueBuilderBase { + // add a dense subspace for the given address (label for all + // mapped dimensions in canonical order). Note that previously + // returned subspaces will be invalidated when new subspaces are + // added. Also note that adding the same subspace multiple times + // is not allowed. + virtual ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) = 0; + + // Given the ownership of the builder itself, produce the newly + // created value. This means that builders can only be used once, + // it also means values can build themselves. + virtual std::unique_ptr<NewValue> build(std::unique_ptr<ValueBuilder> self) = 0; +}; + +/** + * Factory able to create appropriate value builders. We do not really + * care about the full mathematical type here, but it needs to be + * passed since it is exposed in the value api. The expected number of + * subspaces is also passed since it enables the builder to pre-size + * internal structures appropriately. Note that since we are not able + * to have virtual templated functions we need to cast the created + * builder. With interoperability between all values. + **/ +struct ValueBuilderFactory { + template <typename T> + std::unique_ptr<ValueBuilder<T>> create_value_builder(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const + { + assert(check_cell_type<T>(type.cell_type())); + auto base = create_value_builder_base(type, num_mapped_dims_in, subspace_size_in, expected_subspaces); + ValueBuilder<T> *builder = dynamic_cast<ValueBuilder<T>*>(base.get()); + assert(builder); + base.release(); + return std::unique_ptr<ValueBuilder<T>>(builder); + } + template <typename T> + std::unique_ptr<ValueBuilder<T>> create_value_builder(const ValueType &type) const + { + return create_value_builder<T>(type, type.count_mapped_dimensions(), type.dense_subspace_size(), 1); + } + virtual ~ValueBuilderFactory() {} +protected: + virtual std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const = 0; +}; + +/** + * A simple implementation of a generic value that can also be used to + * build new values. This class focuses on simplicity over speed and + * is intended as a reference implementation that can also be used to + * test the correctness of tensor operations as they are moved away + * from the implementation of individual tensor classes. + **/ +class SimpleValue : public NewValue, public NewValue::Index +{ +private: + using Addr = std::vector<vespalib::string>; + ValueType _type; + size_t _num_mapped_dims; + size_t _subspace_size; + std::map<Addr,size_t> _index; +protected: + size_t subspace_size() const { return _subspace_size; } + void add_mapping(const std::vector<vespalib::stringref> &addr); +public: + SimpleValue(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in); + ~SimpleValue() override; + const ValueType &type() const override { return _type; } + const NewValue::Index &index() const override { return *this; } + size_t size() const override { return _index.size(); } + std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; +}; + +/** + * Subclasses of SimpleValue handling cell type specialization. + **/ +template <typename T> +class SimpleValueT : public SimpleValue, public ValueBuilder<T> +{ +private: + std::vector<T> _cells; +public: + SimpleValueT(const ValueType &type, size_t num_mapped_dims_in, size_t subspace_size_in); + ~SimpleValueT() override; + TypedCells cells() const override { return TypedCells(ConstArrayRef<T>(_cells)); } + ArrayRef<T> add_subspace(const std::vector<vespalib::stringref> &addr) override; + std::unique_ptr<NewValue> build(std::unique_ptr<ValueBuilder<T>> self) override { + ValueBuilder<T>* me = this; + assert(me == self.get()); + self.release(); + return std::unique_ptr<NewValue>(this); + } +}; + +/** + * ValueBuilderFactory implementation for SimpleValue. + **/ +struct SimpleValueBuilderFactory : ValueBuilderFactory { + ~SimpleValueBuilderFactory() override {} +protected: + std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, + size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const override; +}; + +/** + * Plan for how to traverse two partially overlapping dense subspaces + * in parallel, identifying all matching cell index combinations, in + * the exact order the joined cells will be stored in the result. The + * plan can be made up-front during tensor function compilation. + **/ +struct DenseJoinPlan { + size_t lhs_size; + size_t rhs_size; + size_t out_size; + std::vector<size_t> loop_cnt; + std::vector<size_t> lhs_stride; + std::vector<size_t> rhs_stride; + DenseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~DenseJoinPlan(); + template <typename F> void execute(size_t lhs, size_t rhs, F &&f) const { + switch(loops_left(0)) { + case 0: return execute_few<F, 0>(0, lhs, rhs, std::forward<F>(f)); + case 1: return execute_few<F, 1>(0, lhs, rhs, std::forward<F>(f)); + case 2: return execute_few<F, 2>(0, lhs, rhs, std::forward<F>(f)); + case 3: return execute_few<F, 3>(0, lhs, rhs, std::forward<F>(f)); + default: return execute_many<F>(0, lhs, rhs, std::forward<F>(f)); + } + } +private: + size_t loops_left(size_t idx) const { return (loop_cnt.size() - idx); } + template <typename F, size_t N> void execute_few(size_t idx, size_t lhs, size_t rhs, F &&f) const { + if constexpr (N == 0) { + f(lhs, rhs); + } else { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + execute_few<F, N - 1>(idx + 1, lhs, rhs, std::forward<F>(f)); + } + } + } + template <typename F> void execute_many(size_t idx, size_t lhs, size_t rhs, F &&f) const { + for (size_t i = 0; i < loop_cnt[idx]; ++i, lhs += lhs_stride[idx], rhs += rhs_stride[idx]) { + if (loops_left(idx + 1) == 3) { + execute_few<F, 3>(idx + 1, lhs, rhs, std::forward<F>(f)); + } else { + execute_many<F>(idx + 1, lhs, rhs, std::forward<F>(f)); + } + } + } +}; + +/** + * Plan for how to join the sparse part (all mapped dimensions) + * between two values. The plan can be made up-front during tensor + * function compilation. + **/ +struct SparseJoinPlan { + enum class Source { LHS, RHS, BOTH }; + std::vector<Source> sources; + std::vector<size_t> lhs_overlap; + std::vector<size_t> rhs_overlap; + SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + ~SparseJoinPlan(); +}; + +/** + * Generic join operation treating both values as mixed + * tensors. Packaging will change, and while the baseline join will + * not have information about low-level value class implementations, + * it will have up-front knowledge about types, specifically + * dimensional overlap and result type. + **/ +using join_fun_t = double (*)(double, double); +std::unique_ptr<NewValue> new_join(const NewValue &a, const NewValue &b, join_fun_t function, const ValueBuilderFactory &factory); + +/** + * Make a value from a tensor spec using a value builder factory + * interface, making it work with any value implementation. + **/ +std::unique_ptr<NewValue> new_value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory); + +/** + * Convert a generic value to a tensor spec. + **/ +TensorSpec spec_from_new_value(const NewValue &value); + +} diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index 2287e8599eb..30a36fcba1d 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -174,6 +174,18 @@ ValueType::is_dense() const } size_t +ValueType::count_mapped_dimensions() const +{ + size_t cnt = 0; + for (const auto &dim : dimensions()) { + if (dim.is_mapped()) { + ++cnt; + } + } + return cnt; +} + +size_t ValueType::dense_subspace_size() const { size_t size = 1; @@ -186,10 +198,21 @@ ValueType::dense_subspace_size() const } std::vector<ValueType::Dimension> -ValueType::nontrivial_dimensions() const { +ValueType::nontrivial_indexed_dimensions() const { + std::vector<ValueType::Dimension> result; + for (const auto &dim: dimensions()) { + if (dim.is_indexed() && !dim.is_trivial()) { + result.push_back(dim); + } + } + return result; +} + +std::vector<ValueType::Dimension> +ValueType::mapped_dimensions() const { std::vector<ValueType::Dimension> result; for (const auto &dim: dimensions()) { - if (!dim.is_trivial()) { + if (dim.is_mapped()) { result.push_back(dim); } } diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index a3dbb901eb0..4199b3a3381 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -60,9 +60,11 @@ public: bool is_tensor() const { return (_type == Type::TENSOR); } bool is_sparse() const; bool is_dense() const; + size_t count_mapped_dimensions() const; size_t dense_subspace_size() const; const std::vector<Dimension> &dimensions() const { return _dimensions; } - std::vector<Dimension> nontrivial_dimensions() const; + std::vector<Dimension> nontrivial_indexed_dimensions() const; + std::vector<Dimension> mapped_dimensions() const; size_t dimension_index(const vespalib::string &name) const; std::vector<vespalib::string> dimension_names() const; bool operator==(const ValueType &rhs) const { diff --git a/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp index 23e54e7fc02..b8832305640 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp @@ -128,8 +128,8 @@ bool check_input_type(const ValueType &type, const DimList &relevant) { } bool is_multi_matmul(const ValueType &a, const ValueType &b, const vespalib::string &reduce_dim) { - auto dims_a = a.nontrivial_dimensions(); - auto dims_b = b.nontrivial_dimensions(); + auto dims_a = a.nontrivial_indexed_dimensions(); + auto dims_b = b.nontrivial_indexed_dimensions(); if (check_input_type(a, dims_a) && check_input_type(b, dims_b) && (a.cell_type() == b.cell_type())) { CommonDim cd_a(dims_a, reduce_dim); CommonDim cd_b(dims_b, reduce_dim); @@ -144,8 +144,8 @@ bool is_multi_matmul(const ValueType &a, const ValueType &b, const vespalib::str const TensorFunction &create_multi_matmul(const TensorFunction &a, const TensorFunction &b, const vespalib::string &reduce_dim, const ValueType &result_type, Stash &stash) { - auto dims_a = a.result_type().nontrivial_dimensions(); - auto dims_b = b.result_type().nontrivial_dimensions(); + auto dims_a = a.result_type().nontrivial_indexed_dimensions(); + auto dims_b = b.result_type().nontrivial_indexed_dimensions(); CommonDim cd_a(dims_a, reduce_dim); CommonDim cd_b(dims_b, reduce_dim); DimPrefix prefix(dims_a, dims_b); diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp index e4e3ffc27d6..e57cfd25325 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp @@ -72,8 +72,8 @@ using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; //----------------------------------------------------------------------------- std::optional<Inner> detect_simple_expand(const TensorFunction &lhs, const TensorFunction &rhs) { - std::vector<ValueType::Dimension> a = lhs.result_type().nontrivial_dimensions(); - std::vector<ValueType::Dimension> b = rhs.result_type().nontrivial_dimensions(); + std::vector<ValueType::Dimension> a = lhs.result_type().nontrivial_indexed_dimensions(); + std::vector<ValueType::Dimension> b = rhs.result_type().nontrivial_indexed_dimensions(); if (a.empty() || b.empty()) { return std::nullopt; } else if (a.back().name < b.front().name) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp index c407ef6cdff..05de3d07c96 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -130,8 +130,8 @@ Primary select_primary(const TensorFunction &lhs, const TensorFunction &rhs, Val } std::optional<Overlap> detect_overlap(const TensorFunction &primary, const TensorFunction &secondary) { - std::vector<ValueType::Dimension> a = primary.result_type().nontrivial_dimensions(); - std::vector<ValueType::Dimension> b = secondary.result_type().nontrivial_dimensions(); + std::vector<ValueType::Dimension> a = primary.result_type().nontrivial_indexed_dimensions(); + std::vector<ValueType::Dimension> b = secondary.result_type().nontrivial_indexed_dimensions(); if (b.size() > a.size()) { return std::nullopt; } else if (b == a) { diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp index 87ab80c2a8e..1538ecdd12f 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp @@ -18,6 +18,9 @@ #include <vespa/vespalib/stllike/hash_map_equal.hpp> #include <vespa/vespalib/util/array_equal.hpp> +#include <vespa/log/log.h> +LOG_SETUP(".eval.tensor.sparse.sparse_tensor"); + using vespalib::eval::TensorSpec; namespace vespalib::tensor { @@ -106,6 +109,21 @@ SparseTensor::equals(const Tensor &arg) const Tensor::UP SparseTensor::clone() const { + size_t mem_use = _stash.get_memory_usage().usedBytes(); + if (mem_use < (STASH_CHUNK_SIZE / 4)) { + size_t aligned_size = (mem_use + 63) & ~(sizeof(char *) - 1); + Stash stash_copy(aligned_size); + Cells cells_copy; + copyCells(cells_copy, _cells, stash_copy); + if (stash_copy.get_memory_usage().allocatedBytes() * 2 > STASH_CHUNK_SIZE) { + LOG(warning, "shrink failed, %zu bytes -> chunksize %zu -> allocated %zu", + mem_use, aligned_size, stash_copy.get_memory_usage().allocatedBytes()); + } + eval::ValueType type_copy = _type; + return std::make_unique<SparseTensor>(std::move(type_copy), + std::move(cells_copy), + std::move(stash_copy)); + } return std::make_unique<SparseTensor>(_type, _cells); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 0da68f576b6..b704516bd2d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -196,6 +196,12 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_DIRECT_STORAGE_API_RPC = defineFeatureFlag( + "use-direct-storage-api-rpc", false, + "Whether to use direct RPC for Storage API communication between content cluster nodes.", + "Takes effect at restart of distributor and content node process", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag HOST_HARDENING = defineFeatureFlag( "host-hardening", false, "Whether to enable host hardening Linux baseline.", @@ -264,11 +270,6 @@ public class Flags { "Whether to provision and use endpoint certs for apps in shared routing zones", "Takes effect on next deployment of the application", APPLICATION_ID); - public static final UnboundBooleanFlag NLB_PROXY_PROTOCOL = defineFeatureFlag( - "nlb-proxy-protocol", false, - "Configure NLB to use proxy protocol", - "Takes effect on next application redeploy", - APPLICATION_ID); public static final UnboundBooleanFlag USE_CLOUD_INIT_FORMAT = defineFeatureFlag( "use-cloud-init", false, diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index bce6f72c1fc..895ddb30a6d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -48,7 +48,6 @@ public class Request extends AbstractResource { private boolean serverRequest; private Long timeout; private URI uri; - private RequestType requestType; public enum RequestType { READ, WRITE, MONITORING @@ -331,12 +330,6 @@ public class Request extends AbstractResource { return unit.convert(creationTime, TimeUnit.MILLISECONDS); } - /** Sets the type classification of this request for metric collection purposes */ - public void setRequestType(RequestType requestType) { this.requestType = requestType; } - - /** Returns the type classification of this request for metric collection purposes, or null if not set */ - public RequestType getRequestType() { return requestType; } - /** * <p>Returns whether or not this Request has been cancelled. This can be thought of as the {@link * Thread#isInterrupted()} of Requests - it does not enforce anything in ways of blocking the Request, it is simply diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java index ec9d2e0df84..c3d07a70e14 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Response.java @@ -102,6 +102,7 @@ public class Response { private final HeaderFields headers = new HeaderFields(); private Throwable error; private int status; + private Request.RequestType requestType; /** * Creates a new instance of this class. @@ -205,6 +206,12 @@ public class Response { return this; } + /** Sets the type classification of this request for metric collection purposes */ + public void setRequestType(Request.RequestType requestType) { this.requestType = requestType; } + + /** Returns the type classification of this request for metric collection purposes, or null if not set */ + public Request.RequestType getRequestType() { return requestType; } + /** * This is a convenience method for creating a Response with status {@link Status#GATEWAY_TIMEOUT} and passing * that to the given {@link ResponseHandler#handleResponse(Response)}. For trivial implementations of {@link diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java index 3dccb09c971..f06f9e256ff 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterRequest.java @@ -18,8 +18,6 @@ import java.util.Set; /** * Servlet implementation for JDisc filter requests. - * - * @since 5.27 */ class ServletFilterRequest extends DiscFilterRequest { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 53d775d4349..7085f07585a 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -44,12 +44,12 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog private final AccessLog accessLog; - public AccessLogRequestLog(final AccessLog accessLog) { + public AccessLogRequestLog(AccessLog accessLog) { this.accessLog = accessLog; } @Override - public void log(final Request request, final Response response) { + public void log(Request request, Response response) { try { AccessLogEntry accessLogEntry = Optional.ofNullable(request.getAttribute(JDiscHttpServlet.ATTRIBUTE_NAME_ACCESS_LOG_ENTRY)) .map(AccessLogEntry.class::cast) @@ -100,8 +100,8 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog accessLogEntry.addKeyValue("cipher-suite", cipherSuite); } - final long startTime = request.getTimeStamp(); - final long endTime = System.currentTimeMillis(); + long startTime = request.getTimeStamp(); + long endTime = System.currentTimeMillis(); accessLogEntry.setTimeStamp(startTime); accessLogEntry.setDurationBetweenRequestResponse(endTime - startTime); accessLogEntry.setReturnedContentSize(response.getHttpChannel().getBytesWritten()); @@ -121,7 +121,7 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog } } - private static String getRemoteAddress(final HttpServletRequest request) { + private static String getRemoteAddress(HttpServletRequest request) { return Optional.ofNullable(request.getHeader(HEADER_NAME_X_FORWARDED_FOR)) .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_Y_RA))) .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_YAHOOREMOTEIP))) @@ -129,7 +129,7 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog .orElseGet(request::getRemoteAddr); } - private static int getRemotePort(final HttpServletRequest request) { + private static int getRemotePort(HttpServletRequest request) { return Optional.ofNullable(request.getHeader(HEADER_NAME_X_FORWARDED_PORT)) .or(() -> Optional.ofNullable(request.getHeader(HEADER_NAME_Y_RP))) .flatMap(AccessLogRequestLog::parsePort) @@ -143,4 +143,5 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog return Optional.empty(); } } + } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 81577561c5b..cc7ed7ac3e0 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -175,9 +175,6 @@ class HttpRequestDispatch { try (ResourceReference ref = References.fromResource(jdiscRequest)) { HttpRequestFactory.copyHeaders(jettyRequest, jdiscRequest); requestContentChannel = requestHandler.handleRequest(jdiscRequest, servletResponseController.responseHandler); - if (jdiscRequest.getRequestType() != null) - jettyRequest.setAttribute(HttpResponseStatisticsCollector.requestTypeAttribute, - jdiscRequest.getRequestType()); } ServletInputStream servletInputStream = jettyRequest.getInputStream(); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java index d2ac0cb7f6a..31a8303ab4b 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java @@ -287,6 +287,15 @@ public class HttpResponseStatisticsCollector extends HandlerWrapper implements G this.value = value; } + @Override + public String toString() { + return "scheme: " + scheme + + ", method: " + method + + ", name: " + name + + ", requestType: " + requestType + + ", value: " + value; + } + } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index ba477f9d32f..b050a9a6d1c 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -338,6 +338,8 @@ public class JettyHttpServer extends AbstractServerProvider { return ((ServerConnector)server.getConnectors()[0]).getLocalPort(); } + Server server() { return server; } + private class MetricTask implements Runnable { @Override public void run() { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java index 0188e7c2f09..5dd6b72dc20 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java @@ -164,6 +164,7 @@ public class ServletResponseController { private void setResponse(Response jdiscResponse) { synchronized (monitor) { + servletRequest.setAttribute(HttpResponseStatisticsCollector.requestTypeAttribute, jdiscResponse.getRequestType()); if (responseCommitted) { log.log(Level.FINE, jdiscResponse.getError(), diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java index eeae1fa74bc..a3cb31d5ecb 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java @@ -28,7 +28,7 @@ public class HttpRequestTestCase { @Test public void requireThatSimpleServerConstructorsUseReasonableDefaults() { - final URI uri = URI.create("http://localhost/"); + URI uri = URI.create("http://localhost/"); HttpRequest request = HttpRequest.newServerRequest(mockContainer(), uri); assertTrue(request.isServerRequest()); assertEquals(uri, request.getUri()); @@ -50,9 +50,9 @@ public class HttpRequestTestCase { @Test public void requireThatSimpleClientConstructorsUseReasonableDefaults() { - final Request parent = new Request(mockContainer(), URI.create("http://localhost/")); + Request parent = new Request(mockContainer(), URI.create("http://localhost/")); - final URI uri = URI.create("http://remotehost/"); + URI uri = URI.create("http://remotehost/"); HttpRequest request = HttpRequest.newClientRequest(parent, uri); assertFalse(request.isServerRequest()); assertEquals(uri, request.getUri()); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index a67f6919727..96cf1d4c01f 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -40,6 +40,7 @@ import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V1; import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2; import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.server.handler.AbstractHandlerContainer; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.junit.Rule; import org.junit.Test; @@ -97,13 +98,12 @@ import static org.cthul.matchers.CthulMatchers.matchesPattern; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.mock; @@ -125,8 +125,8 @@ public class HttpServerTest { @Test public void requireThatServerCanListenToRandomPort() throws Exception { final TestDriver driver = TestDrivers.newInstance(mockRequestHandler()); - assertThat(driver.server().getListenPort(), is(not(0))); - assertThat(driver.close(), is(true)); + assertNotEquals(0, driver.server().getListenPort()); + assertTrue(driver.close()); } @Test @@ -142,7 +142,7 @@ public class HttpServerTest { } catch (final Throwable t) { assertThat(t.getCause(), instanceOf(BindException.class)); } - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -159,7 +159,7 @@ public class HttpServerTest { Pattern.quote(BindingSetNotFoundException.class.getName()) + ": No binding set named 'unknown'\\.\n\tat .+", Pattern.DOTALL | Pattern.MULTILINE))); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -171,7 +171,7 @@ public class HttpServerTest { .requestHeaderSize(1)); driver.client().get("/status.html") .expectStatusCode(is(REQUEST_URI_TOO_LONG)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -188,7 +188,7 @@ public class HttpServerTest { assertThat(accessLogMock.logEntries.size(), equalTo(1)); AccessLogEntry accessLogEntry = accessLogMock.logEntries.get(0); - assertThat(accessLogEntry.getStatusCode(), equalTo(414)); + assertEquals(414, accessLogEntry.getStatusCode()); } private static class AccessLogMock extends AccessLog { @@ -208,7 +208,7 @@ public class HttpServerTest { final TestDriver driver = TestDrivers.newInstance(new EchoRequestHandler()); driver.client().get("/status.html") .expectStatusCode(is(OK)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -217,7 +217,7 @@ public class HttpServerTest { SimpleHttpClient client = driver.newClient(true); client.get("/status.html") .expectStatusCode(is(OK)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -227,7 +227,7 @@ public class HttpServerTest { .expectStatusCode(is(OK)); driver.client().get("/status.html") .expectStatusCode(is(OK)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -241,7 +241,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(startsWith('{' + requestContent + "=[]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -254,7 +254,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(is("{foo=[bar]}foo=bar")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -267,7 +267,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(is("{foo=[bar]}foo=bar")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -280,7 +280,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(is("{foo=[bar]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -295,7 +295,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(startsWith('{' + requestContent + "=[]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -307,7 +307,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(is("{}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -320,7 +320,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(startsWith("{a=[b], c=[d]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -332,7 +332,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(is("{a=[b], c=[d]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -345,7 +345,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(startsWith("{a=[b], c=[d1, d2], e=[f]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -358,7 +358,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(is("{B\u00e6r=[bl\u00e5]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -370,7 +370,7 @@ public class HttpServerTest { .setContent("a=b") .execute(); response.expectStatusCode(is(UNSUPPORTED_MEDIA_TYPE)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -383,7 +383,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(startsWith("{ =\u00d8=[\"% ]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -395,7 +395,7 @@ public class HttpServerTest { .setContent("a=b") .execute(); response.expectStatusCode(is(INTERNAL_SERVER_ERROR)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -426,7 +426,7 @@ public class HttpServerTest { .execute(); response.expectStatusCode(is(OK)) .expectContent(containsString("[foo=bar]")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -441,7 +441,7 @@ public class HttpServerTest { .expectStatusCode(is(OK)) .expectHeader("Set-Cookie", is("foo=bar; Path=/foopath; Domain=.localhost; Secure; HttpOnly")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -451,7 +451,7 @@ public class HttpServerTest { driver.client().get("/status.html") .expectStatusCode(is(GATEWAY_TIMEOUT)); ResponseDispatch.newInstance(OK).dispatch(requestHandler.responseHandler); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } // Header with no value is disallowed by https://tools.ietf.org/html/rfc7230#section-3.2 @@ -462,7 +462,7 @@ public class HttpServerTest { driver.client().get("/status.html") .expectStatusCode(is(OK)) .expectNoHeader("X-Foo"); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } // Header with empty value is allowed by https://tools.ietf.org/html/rfc7230#section-3.2 @@ -473,7 +473,7 @@ public class HttpServerTest { driver.client().get("/status.html") .expectStatusCode(is(OK)) .expectHeader("X-Foo", is("")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -498,7 +498,7 @@ public class HttpServerTest { driver.client().get("/status.html") .expectStatusCode(is(OK)) .expectHeader(CONNECTION, is(CLOSE)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -510,7 +510,7 @@ public class HttpServerTest { final TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile, TlsClientAuth.WANT); driver.client().get("/status.html") .expectStatusCode(is(OK)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -528,7 +528,7 @@ public class HttpServerTest { .get("/dummy.html") .expectStatusCode(is(UNAUTHORIZED)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -546,7 +546,7 @@ public class HttpServerTest { .get("/status.html") .expectStatusCode(is(OK)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -560,21 +560,76 @@ public class HttpServerTest { @Test public void requireThatGzipEncodingRequestsAreAutomaticallyDecompressed() throws Exception { - final TestDriver driver = TestDrivers.newInstance(new ParameterPrinterRequestHandler()); - final String requestContent = generateContent('a', 30); - final ResponseValidator response = - driver.client().newPost("/status.html") - .addHeader(CONTENT_TYPE, APPLICATION_X_WWW_FORM_URLENCODED) - .setGzipContent(requestContent) - .execute(); + TestDriver driver = TestDrivers.newInstance(new ParameterPrinterRequestHandler()); + String requestContent = generateContent('a', 30); + ResponseValidator response = driver.client().newPost("/status.html") + .addHeader(CONTENT_TYPE, APPLICATION_X_WWW_FORM_URLENCODED) + .setGzipContent(requestContent) + .execute(); response.expectStatusCode(is(OK)) .expectContent(startsWith('{' + requestContent + "=[]}")); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); + } + + @Test + public void requireThatResponseStatsAreCollected() throws Exception { + RequestTypeHandler handler = new RequestTypeHandler(); + TestDriver driver = TestDrivers.newInstance(handler); + HttpResponseStatisticsCollector statisticsCollector = ((AbstractHandlerContainer) driver.server().server().getHandler()) + .getChildHandlerByClass(HttpResponseStatisticsCollector.class); + + { + List<HttpResponseStatisticsCollector.StatisticsEntry> stats = statisticsCollector.takeStatistics(); + assertEquals(0, stats.size()); + } + + { + driver.client().newPost("/status.html").execute(); + var entry = waitForStatistics(statisticsCollector); + assertEquals("http", entry.scheme); + assertEquals("POST", entry.method); + assertEquals("http.status.2xx", entry.name); + assertEquals("write", entry.requestType); + assertEquals(1, entry.value); + } + + { + driver.client().newGet("/status.html").execute(); + var entry = waitForStatistics(statisticsCollector); + assertEquals("http", entry.scheme); + assertEquals("GET", entry.method); + assertEquals("http.status.2xx", entry.name); + assertEquals("read", entry.requestType); + assertEquals(1, entry.value); + } + + { + handler.setRequestType(Request.RequestType.READ); + driver.client().newPost("/status.html").execute(); + var entry = waitForStatistics(statisticsCollector); + assertEquals("Handler overrides request type", "read", entry.requestType); + } + + assertTrue(driver.close()); + } + + private HttpResponseStatisticsCollector.StatisticsEntry waitForStatistics(HttpResponseStatisticsCollector + statisticsCollector) { + List<HttpResponseStatisticsCollector.StatisticsEntry> entries = Collections.emptyList(); + int tries = 0; + while (entries.isEmpty() && tries < 10000) { + entries = statisticsCollector.takeStatistics(); + if (entries.isEmpty()) + try {Thread.sleep(100); } catch (InterruptedException e) {} + tries++; + } + assertEquals(1, entries.size()); + return entries.get(0); } @Test public void requireThatConnectionThrottleDoesNotBlockConnectionsBelowThreshold() throws Exception { - final TestDriver driver = TestDrivers.newConfiguredInstance( + TestDriver driver = TestDrivers.newConfiguredInstance( new EchoRequestHandler(), new ServerConfig.Builder(), new ConnectorConfig.Builder() @@ -585,7 +640,7 @@ public class HttpServerTest { .maxConnections(10))); driver.client().get("/status.html") .expectStatusCode(is(OK)); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -603,7 +658,7 @@ public class HttpServerTest { driver, clientCtx, null, null, "Received fatal alert: bad_certificate"); verify(metricConsumer.mockitoMock()) .add(Metrics.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -623,7 +678,7 @@ public class HttpServerTest { driver, clientCtx, "TLSv1.3", null, "Received fatal alert: protocol_version"); verify(metricConsumer.mockitoMock()) .add(Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -643,7 +698,7 @@ public class HttpServerTest { driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure"); verify(metricConsumer.mockitoMock()) .add(Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -667,7 +722,7 @@ public class HttpServerTest { driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); verify(metricConsumer.mockitoMock()) .add(Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -690,7 +745,7 @@ public class HttpServerTest { driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); verify(metricConsumer.mockitoMock()) .add(Metrics.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); } @Test @@ -707,9 +762,9 @@ public class HttpServerTest { sendJettyClientRequest(driver, client, new V1.Tag(proxiedRemoteAddress, proxiedRemotePort)); sendJettyClientRequest(driver, client, new V2.Tag(proxiedRemoteAddress, proxiedRemotePort)); client.stop(); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); - assertThat(accessLogMock.logEntries, hasSize(2)); + assertEquals(2, accessLogMock.logEntries.size()); assertLogEntryHasRemote(accessLogMock.logEntries.get(0), proxiedRemoteAddress, proxiedRemotePort); assertLogEntryHasRemote(accessLogMock.logEntries.get(1), proxiedRemoteAddress, proxiedRemotePort); } @@ -727,9 +782,9 @@ public class HttpServerTest { sendJettyClientRequest(driver, client, null); sendJettyClientRequest(driver, client, new V2.Tag(proxiedRemoteAddress, 12345)); client.stop(); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); - assertThat(accessLogMock.logEntries, hasSize(2)); + assertEquals(2, accessLogMock.logEntries.size()); assertLogEntryHasRemote(accessLogMock.logEntries.get(0), "127.0.0.1", 0); assertLogEntryHasRemote(accessLogMock.logEntries.get(1), proxiedRemoteAddress, 0); } @@ -751,7 +806,7 @@ public class HttpServerTest { proxiedRemoteAddress, proxiedRemotePort, proxyLocalAddress, proxyLocalPort, null); ContentResponse response = sendJettyClientRequest(driver, client, v2Tag); client.stop(); - assertThat(driver.close(), is(true)); + assertTrue(driver.close()); int clientPort = Integer.parseInt(response.getHeaders().get("Jdisc-Local-Port")); assertNotEquals(proxyLocalPort, clientPort); @@ -879,8 +934,8 @@ public class HttpServerTest { return ret.toString(); } - private static TestDriver newDriverWithFormPostContentRemoved( - final RequestHandler requestHandler, final boolean removeFormPostBody) throws Exception { + private static TestDriver newDriverWithFormPostContentRemoved(RequestHandler requestHandler, + boolean removeFormPostBody) throws Exception { return TestDrivers.newConfiguredInstance( requestHandler, new ServerConfig.Builder() @@ -888,8 +943,7 @@ public class HttpServerTest { new ConnectorConfig.Builder()); } - private static FormBodyPart newFileBody(final String parameterName, final String fileName, final String fileContent) - throws Exception { + private static FormBodyPart newFileBody(final String parameterName, final String fileName, final String fileContent) { return new FormBodyPart( parameterName, new StringBody(fileContent, ContentType.TEXT_PLAIN) { @@ -959,23 +1013,37 @@ public class HttpServerTest { } private static class ParameterPrinterRequestHandler extends AbstractRequestHandler { + private static final CompletionHandler NULL_COMPLETION_HANDLER = null; @Override - public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { - final Map<String, List<String>> parameters = - new TreeMap<>(((HttpRequest)request).parameters()); - final ContentChannel responseContentChannel - = ResponseDispatch.newInstance(Response.Status.OK).connect(handler); - responseContentChannel.write( - ByteBuffer.wrap(parameters.toString().getBytes(StandardCharsets.UTF_8)), - NULL_COMPLETION_HANDLER); + public ContentChannel handleRequest(Request request, ResponseHandler handler) { + Map<String, List<String>> parameters = new TreeMap<>(((HttpRequest)request).parameters()); + ContentChannel responseContentChannel = ResponseDispatch.newInstance(Response.Status.OK).connect(handler); + responseContentChannel.write(ByteBuffer.wrap(parameters.toString().getBytes(StandardCharsets.UTF_8)), + NULL_COMPLETION_HANDLER); // Have the request content written back to the response. return responseContentChannel; } } + private static class RequestTypeHandler extends AbstractRequestHandler { + + private Request.RequestType requestType = null; + + public void setRequestType(Request.RequestType requestType) { + this.requestType = requestType; + } + + @Override + public ContentChannel handleRequest(Request request, ResponseHandler handler) { + Response response = new Response(OK); + response.setRequestType(requestType); + return handler.handleResponse(response); + } + } + private static class ThrowingHandler extends AbstractRequestHandler { @Override public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java index 8035734a76c..f1d710bd10f 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; +import com.yahoo.jdisc.Request; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -36,6 +37,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertNotNull; /** * A simple http client for testing @@ -87,23 +89,23 @@ public class SimpleHttpClient implements AutoCloseable { return URI.create(scheme + "://localhost:" + listenPort + path); } - public RequestExecutor newGet(final String path) { + public RequestExecutor newGet(String path) { return newRequest(new HttpGet(newUri(path))); } - public RequestExecutor newPost(final String path) { + public RequestExecutor newPost(String path) { return newRequest(new HttpPost(newUri(path))); } - public RequestExecutor newRequest(final HttpUriRequest request) { + public RequestExecutor newRequest(HttpUriRequest request) { return new RequestExecutor().setRequest(request); } - public ResponseValidator execute(final HttpUriRequest request) throws IOException { + public ResponseValidator execute(HttpUriRequest request) throws IOException { return newRequest(request).execute(); } - public ResponseValidator get(final String path) throws IOException { + public ResponseValidator get(String path) throws IOException { return newGet(path).execute(); } @@ -164,37 +166,37 @@ public class SimpleHttpClient implements AutoCloseable { private final HttpResponse response; private final String content; - public ResponseValidator(final HttpResponse response) throws IOException { + public ResponseValidator(HttpResponse response) throws IOException { this.response = response; - final HttpEntity entity = response.getEntity(); - this.content = entity == null ? null : - EntityUtils.toString(entity, StandardCharsets.UTF_8); + HttpEntity entity = response.getEntity(); + this.content = entity == null ? null : EntityUtils.toString(entity, StandardCharsets.UTF_8); } - public ResponseValidator expectStatusCode(final Matcher<Integer> matcher) { + public ResponseValidator expectStatusCode(Matcher<Integer> matcher) { MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), matcher); return this; } - public ResponseValidator expectHeader(final String headerName, final Matcher<String> matcher) { - final Header firstHeader = response.getFirstHeader(headerName); - final String headerValue = firstHeader != null ? firstHeader.getValue() : null; + public ResponseValidator expectHeader(String headerName, Matcher<String> matcher) { + Header firstHeader = response.getFirstHeader(headerName); + String headerValue = firstHeader != null ? firstHeader.getValue() : null; MatcherAssert.assertThat(headerValue, matcher); - assertThat(firstHeader, is(not(nullValue()))); + assertNotNull(firstHeader); return this; } - public ResponseValidator expectNoHeader(final String headerName) { - final Header firstHeader = response.getFirstHeader(headerName); + public ResponseValidator expectNoHeader(String headerName) { + Header firstHeader = response.getFirstHeader(headerName); assertThat(firstHeader, is(nullValue())); return this; } - public ResponseValidator expectContent(final Matcher<String> matcher) throws IOException { + public ResponseValidator expectContent(final Matcher<String> matcher) { MatcherAssert.assertThat(content, matcher); return this; } } + } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java index ed571c6b07a..913b22bc0e3 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDriver.java @@ -24,8 +24,7 @@ public class TestDriver { private final JettyHttpServer server; private final SimpleHttpClient client; - private TestDriver(com.yahoo.jdisc.test.TestDriver driver, JettyHttpServer server, SimpleHttpClient client) - throws IOException { + private TestDriver(com.yahoo.jdisc.test.TestDriver driver, JettyHttpServer server, SimpleHttpClient client) { this.driver = driver; this.server = server; this.client = client; @@ -33,7 +32,7 @@ public class TestDriver { public static TestDriver newInstance(Class<? extends JettyHttpServer> serverClass, RequestHandler requestHandler, - Module testConfig) throws IOException { + Module testConfig) { com.yahoo.jdisc.test.TestDriver driver = com.yahoo.jdisc.test.TestDriver.newSimpleApplicationInstance(testConfig); ContainerBuilder builder = driver.newContainerBuilder(); @@ -47,7 +46,7 @@ public class TestDriver { return new TestDriver(driver, server, client); } - public boolean close() throws IOException { + public boolean close() { server.close(); server.release(); return driver.close(); @@ -65,7 +64,7 @@ public class TestDriver { return newSslContext(driver.newContainerBuilder()); } - private static SSLContext newSslContext(final ContainerBuilder builder) { + private static SSLContext newSslContext(ContainerBuilder builder) { ConnectorConfig.Ssl sslConfig = builder.getInstance(ConnectorConfig.class).ssl(); if (!sslConfig.enabled()) return null; diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java index 4908da2ba75..255e42fb886 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java @@ -24,18 +24,17 @@ import java.nio.file.Path; */ public class TestDrivers { - public static TestDriver newConfiguredInstance(final RequestHandler requestHandler, - final ServerConfig.Builder serverConfig, - final ConnectorConfig.Builder connectorConfig, - final Module... guiceModules) throws IOException { + public static TestDriver newConfiguredInstance(RequestHandler requestHandler, + ServerConfig.Builder serverConfig, + ConnectorConfig.Builder connectorConfig, + Module... guiceModules) throws IOException { return TestDriver.newInstance( JettyHttpServer.class, requestHandler, newConfigModule(serverConfig, connectorConfig, guiceModules)); } - public static TestDriver newInstance(final RequestHandler requestHandler, - final Module... guiceModules) throws IOException { + public static TestDriver newInstance(RequestHandler requestHandler, Module... guiceModules) throws IOException { return TestDriver.newInstance( JettyHttpServer.class, requestHandler, @@ -48,11 +47,11 @@ public class TestDrivers { public enum TlsClientAuth { NEED, WANT } - public static TestDriver newInstanceWithSsl(final RequestHandler requestHandler, + public static TestDriver newInstanceWithSsl(RequestHandler requestHandler, Path certificateFile, Path privateKeyFile, TlsClientAuth tlsClientAuth, - final Module... guiceModules) throws IOException { + Module... guiceModules) throws IOException { return TestDriver.newInstance( JettyHttpServer.class, requestHandler, @@ -74,10 +73,9 @@ public class TestDrivers { Modules.combine(guiceModules))); } - private static Module newConfigModule( - final ServerConfig.Builder serverConfig, - final ConnectorConfig.Builder connectorConfigBuilder, - final Module... guiceModules) { + private static Module newConfigModule(ServerConfig.Builder serverConfig, + ConnectorConfig.Builder connectorConfigBuilder, + Module... guiceModules) { return Modules.combine( new AbstractModule() { @Override @@ -87,12 +85,13 @@ public class TestDrivers { bind(ConnectorConfig.class).toInstance(new ConnectorConfig(connectorConfigBuilder)); bind(FilterBindings.class).toInstance( new FilterBindings( - new BindingRepository<RequestFilter>(), - new BindingRepository<ResponseFilter>())); + new BindingRepository<>(), + new BindingRepository<>())); } }, new ConnectorFactoryRegistryModule(connectorConfigBuilder), new ServletModule(), Modules.combine(guiceModules)); } + } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index c04dca465a1..b895d6221c3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.logging.Logger; import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID; @@ -31,14 +30,11 @@ import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static com.google.common.base.Strings.isNullOrEmpty; /** - * @author Unknown * @author gjoranv */ public class VespaMetrics { - private static final Logger log = Logger.getLogger(VespaMetrics.class.getPackage().getName()); - - public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa"); + public static final ConsumerId vespaMetricsConsumerId = toConsumerId("Vespa"); public static final DimensionId METRIC_TYPE_DIMENSION_ID = toDimensionId("metrictype"); public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId(INTERNAL_SERVICE_ID); @@ -68,8 +64,8 @@ public class VespaMetrics { } /** - * @param services The services to get metrics for - * @return A list of metrics packet builders (to allow modification by the caller). + * @param services the services to get metrics for + * @return a list of metrics packet builders (to allow modification by the caller) */ public List<MetricsPacket.Builder> getMetrics(List<VespaService> services) { List<MetricsPacket.Builder> metricsPackets = new ArrayList<>(); @@ -253,7 +249,7 @@ public class VespaMetrics { String alias = key; boolean isForwarded = false; - for (ConsumersConfig.Consumer.Metric metricConsumer : getMetricDefinitions(VESPA_CONSUMER_ID)) { + for (ConsumersConfig.Consumer.Metric metricConsumer : getMetricDefinitions(vespaMetricsConsumerId)) { if (metricConsumer.name().equals(key)) { alias = metricConsumer.outputname(); isForwarded = true; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 51bdae1aab3..cf2f6210f39 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -24,7 +24,7 @@ public class ValuesFetcher { private static final Logger log = Logger.getLogger(ValuesFetcher.class.getName()); - public static final ConsumerId DEFAULT_PUBLIC_CONSUMER_ID = toConsumerId("default"); + public static final ConsumerId defaultMetricsConsumerId = toConsumerId("default"); private final MetricsManager metricsManager; private final VespaServices vespaServices; @@ -62,12 +62,12 @@ public class ValuesFetcher { } public static ConsumerId getConsumerOrDefault(String requestedConsumer, MetricsConsumers consumers) { - if (requestedConsumer == null) return DEFAULT_PUBLIC_CONSUMER_ID; + if (requestedConsumer == null) return defaultMetricsConsumerId; ConsumerId consumerId = toConsumerId(requestedConsumer); if (! consumers.getAllConsumers().contains(consumerId)) { log.info("No consumer with id '" + requestedConsumer + "' - using the default consumer instead."); - return DEFAULT_PUBLIC_CONSUMER_ID; + return defaultMetricsConsumerId; } return consumerId; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index 5c7e64c4ed1..15f924505be 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.stream.Collectors; -import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; +import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toMap; @@ -50,7 +50,6 @@ public class ApplicationMetricsRetriever extends AbstractComponent { // Non-final for testing private Duration taskTimeout; - @Inject public ApplicationMetricsRetriever(MetricsNodesConfig nodesConfig) { clients = createNodeClients(nodesConfig); @@ -66,7 +65,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent { } public Map<Node, List<MetricsPacket>> getMetrics() { - return getMetrics(DEFAULT_PUBLIC_CONSUMER_ID); + return getMetrics(defaultMetricsConsumerId); } public Map<Node, List<MetricsPacket>> getMetrics(ConsumerId consumer) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index 8d5a1f50918..2d5cd9acb5d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -29,6 +29,7 @@ import static java.util.stream.Collectors.joining; * @author gjoranv */ public class MetricsPacket { + public final int statusCode; public final String statusMessage; public final long timestamp; @@ -80,7 +81,8 @@ public class MetricsPacket { } public static class Builder { - // Set sensible defaults here, and use null guard in all setters. + + // Set defaults here, and use null guard in all setters. // Except for 'service' for which we require an explicit non-null value. private ServiceId service; private int statusCode = 0; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java index d2838d5b1d2..d71b11a4ff2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java @@ -16,7 +16,7 @@ import java.util.Map; import java.util.logging.Logger; import java.util.stream.Collectors; -import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; +import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.Collections.emptyList; @@ -123,7 +123,7 @@ public class YamasJsonUtil { private static YamasJsonModel.YamasJsonNamespace toYamasJsonNamespaces(Collection<ConsumerId> consumers) { YamasJsonModel.YamasJsonNamespace namespaces = new YamasJsonModel.YamasJsonNamespace(); namespaces.namespaces = consumers.stream() - .filter(consumerId -> consumerId != DEFAULT_PUBLIC_CONSUMER_ID) + .filter(consumerId -> consumerId != defaultMetricsConsumerId) .map(consumer -> consumer.id) .collect(Collectors.toList()); return namespaces; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java index 923fb4d646d..a11acf07156 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/ServiceHealthGatherer.java @@ -12,13 +12,11 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; - /** * @author olaa */ public class ServiceHealthGatherer { - protected static List<MetricsPacket.Builder> gatherServiceHealthMetrics(VespaServices vespaServices) { return vespaServices.getVespaServices() .stream() @@ -33,4 +31,5 @@ public class ServiceHealthGatherer { ) .collect(Collectors.toList()); } + } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java index e067639023d..27ac3bbab01 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java @@ -35,4 +35,5 @@ public class TestUtil { } return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n")); } + } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java index f21774aeb15..fd1961fc168 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java @@ -28,7 +28,7 @@ import java.util.Map; import static ai.vespa.metricsproxy.core.MetricsManager.VESPA_VERSION; import static ai.vespa.metricsproxy.core.VespaMetrics.METRIC_TYPE_DIMENSION_ID; -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; +import static ai.vespa.metricsproxy.core.VespaMetrics.vespaMetricsConsumerId; import static ai.vespa.metricsproxy.metric.ExternalMetrics.ROLE_DIMENSION; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; @@ -241,7 +241,7 @@ public class MetricsManagerTest { return new MetricsConsumers(new ConsumersConfig.Builder() .consumer(new Consumer.Builder() - .name(VESPA_CONSUMER_ID.id) + .name(vespaMetricsConsumerId.id) .metric(new Consumer.Metric.Builder() .name(WHITELISTED_METRIC_ID) .outputname(WHITELISTED_METRIC_ID)) diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java index d776368687d..ab586334baa 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java @@ -22,7 +22,7 @@ import com.yahoo.container.jdisc.RequestHandlerTestDriver; import java.time.Instant; import java.util.List; -import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; +import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID; import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; @@ -71,7 +71,7 @@ public class HttpHandlerTestBase { return new MetricsConsumers(new ConsumersConfig.Builder() .consumer(new ConsumersConfig.Consumer.Builder() - .name(DEFAULT_PUBLIC_CONSUMER_ID.id) + .name(defaultMetricsConsumerId.id) .metric(new ConsumersConfig.Consumer.Metric.Builder() .name(CPU_METRIC) .outputname(CPU_METRIC)) diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index 0fa6fea7d11..d7576718e8a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -12,7 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.yahoo.container.jdisc.RequestHandlerTestDriver; import java.util.regex.Pattern; -import java.util.stream.Collectors; + import org.json.JSONArray; import org.json.JSONObject; import org.junit.Before; @@ -26,7 +26,7 @@ import java.util.Map; import java.util.concurrent.Executors; import static ai.vespa.metricsproxy.TestUtil.getFileContents; -import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; +import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.METRICS_V1_PATH; import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.METRICS_VALUES_PATH; import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.PROMETHEUS_VALUES_PATH; @@ -89,7 +89,7 @@ public class ApplicationMetricsHandlerTest { private void setupWireMock() { port = wireMockRule.port(); wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) - .withQueryParam("consumer", equalTo(DEFAULT_PUBLIC_CONSUMER_ID.id)) + .withQueryParam("consumer", equalTo(defaultMetricsConsumerId.id)) .willReturn(aResponse().withBody(RESPONSE))); // Add a slightly different response for a custom consumer. @@ -132,7 +132,7 @@ public class ApplicationMetricsHandlerTest { @Test public void response_contains_node() { - GenericApplicationModel jsonModel = getResponseAsJsonModel(DEFAULT_PUBLIC_CONSUMER_ID.id); + GenericApplicationModel jsonModel = getResponseAsJsonModel(defaultMetricsConsumerId.id); assertEquals(1, jsonModel.nodes.size()); GenericJsonModel nodeModel = jsonModel.nodes.get(0); @@ -161,7 +161,7 @@ public class ApplicationMetricsHandlerTest { @Test public void response_contains_services_with_metrics() { - GenericApplicationModel jsonModel = getResponseAsJsonModel(DEFAULT_PUBLIC_CONSUMER_ID.id); + GenericApplicationModel jsonModel = getResponseAsJsonModel(defaultMetricsConsumerId.id); GenericJsonModel nodeModel = jsonModel.nodes.get(0); assertEquals(2, nodeModel.services.size()); @@ -174,7 +174,7 @@ public class ApplicationMetricsHandlerTest { @Test public void metrics_processors_are_applied() { - GenericApplicationModel jsonModel = getResponseAsJsonModel(DEFAULT_PUBLIC_CONSUMER_ID.id); + GenericApplicationModel jsonModel = getResponseAsJsonModel(defaultMetricsConsumerId.id); GenericService searchnode = jsonModel.nodes.get(0).services.get(0); Map<String, String> dimensions = searchnode.metrics.get(0).dimensions; @@ -233,7 +233,7 @@ public class ApplicationMetricsHandlerTest { private static MetricsConsumers getMetricsConsumers() { return new MetricsConsumers(new ConsumersConfig.Builder() .consumer(new ConsumersConfig.Consumer.Builder() - .name(DEFAULT_PUBLIC_CONSUMER_ID.id)) + .name(defaultMetricsConsumerId.id)) .consumer(new ConsumersConfig.Consumer.Builder() .name(CUSTOM_CONSUMER)) .build()); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java index d8443ece8e8..eba32941620 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java @@ -16,7 +16,7 @@ import java.net.URI; import java.util.List; import static ai.vespa.metricsproxy.TestUtil.getFileContents; -import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; +import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; @@ -56,12 +56,12 @@ public class NodeMetricsClientTest { @BeforeClass public static void setupWireMock() { node = new Node("id", "localhost", wireMockRule.port(), MetricsV1Handler.VALUES_PATH); - URI metricsUri = node.metricsUri(DEFAULT_PUBLIC_CONSUMER_ID); + URI metricsUri = node.metricsUri(defaultMetricsConsumerId); wireMockRule.stubFor(get(urlPathEqualTo(metricsUri.getPath())) .willReturn(aResponse().withBody(RESPONSE))); wireMockRule.stubFor(get(urlPathEqualTo(metricsUri.getPath())) - .withQueryParam("consumer", equalTo(DEFAULT_PUBLIC_CONSUMER_ID.id)) + .withQueryParam("consumer", equalTo(defaultMetricsConsumerId.id)) .willReturn(aResponse().withBody(RESPONSE))); // Add a slightly different response for a custom consumer. @@ -85,34 +85,34 @@ public class NodeMetricsClientTest { @Test public void metrics_are_retrieved_upon_first_request() { - List<MetricsPacket> metrics = nodeMetricsClient.getMetrics(DEFAULT_PUBLIC_CONSUMER_ID); + List<MetricsPacket> metrics = nodeMetricsClient.getMetrics(defaultMetricsConsumerId); assertEquals(1, nodeMetricsClient.snapshotsRetrieved()); assertEquals(4, metrics.size()); } @Test public void cached_metrics_are_used_when_ttl_has_not_expired() { - nodeMetricsClient.getMetrics(DEFAULT_PUBLIC_CONSUMER_ID); + nodeMetricsClient.getMetrics(defaultMetricsConsumerId); assertEquals(1, nodeMetricsClient.snapshotsRetrieved()); clock.advance(NodeMetricsClient.METRICS_TTL.minusMillis(1)); - nodeMetricsClient.getMetrics(DEFAULT_PUBLIC_CONSUMER_ID); + nodeMetricsClient.getMetrics(defaultMetricsConsumerId); assertEquals(1, nodeMetricsClient.snapshotsRetrieved()); } @Test public void metrics_are_refreshed_when_ttl_has_expired() { - nodeMetricsClient.getMetrics(DEFAULT_PUBLIC_CONSUMER_ID); + nodeMetricsClient.getMetrics(defaultMetricsConsumerId); assertEquals(1, nodeMetricsClient.snapshotsRetrieved()); clock.advance(NodeMetricsClient.METRICS_TTL.plusMillis(1)); - nodeMetricsClient.getMetrics(DEFAULT_PUBLIC_CONSUMER_ID); + nodeMetricsClient.getMetrics(defaultMetricsConsumerId); assertEquals(2, nodeMetricsClient.snapshotsRetrieved()); } @Test public void metrics_for_different_consumers_are_cached_separately() { - List<MetricsPacket> defaultMetrics = nodeMetricsClient.getMetrics(DEFAULT_PUBLIC_CONSUMER_ID); + List<MetricsPacket> defaultMetrics = nodeMetricsClient.getMetrics(defaultMetricsConsumerId); assertEquals(1, nodeMetricsClient.snapshotsRetrieved()); assertEquals(4, defaultMetrics.size()); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java index d3a5622d263..67430e50e10 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtilTest.java @@ -7,8 +7,8 @@ import org.junit.Test; import java.util.List; import java.util.Set; -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; -import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; +import static ai.vespa.metricsproxy.core.VespaMetrics.vespaMetricsConsumerId; +import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.YAMAS_ROUTING; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toMetricsPackets; @@ -60,18 +60,18 @@ public class YamasJsonUtilTest { @Test public void default_public_consumer_is_filtered_from_yamas_routing() { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .addConsumers(Set.of(VESPA_CONSUMER_ID, DEFAULT_PUBLIC_CONSUMER_ID)) + .addConsumers(Set.of(vespaMetricsConsumerId, defaultMetricsConsumerId)) .build(); YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet)).metrics.get(0); List<String> namespaces = jsonModel.routing.get(YAMAS_ROUTING).namespaces; assertEquals(1, namespaces.size()); - assertEquals(VESPA_CONSUMER_ID.id, namespaces.get(0)); + assertEquals(vespaMetricsConsumerId.id, namespaces.get(0)); } @Test public void only_default_public_consumer_yields_null_routing_in_json_model() { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .addConsumers(Set.of(DEFAULT_PUBLIC_CONSUMER_ID)) + .addConsumers(Set.of(defaultMetricsConsumerId)) .build(); YamasJsonModel jsonModel = YamasJsonUtil.toYamasArray(singleton(packet)).metrics.get(0); assertNull(jsonModel.routing); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java index 7275cb737c4..9dcf2d9c375 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java @@ -22,7 +22,7 @@ import ai.vespa.metricsproxy.service.VespaServicesConfig.Service; import java.io.IOException; -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; +import static ai.vespa.metricsproxy.core.VespaMetrics.vespaMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; import static ai.vespa.metricsproxy.service.HttpMetricFetcher.STATE_PATH; @@ -88,7 +88,7 @@ public class IntegrationTester implements AutoCloseable { private ConsumersConfig consumersConfig() { return new ConsumersConfig.Builder() - .consumer(createConsumer(VESPA_CONSUMER_ID, "foo.count", "foo_count")) + .consumer(createConsumer(vespaMetricsConsumerId, "foo.count", "foo_count")) .consumer(createConsumer(CUSTOM_CONSUMER_ID, "foo.count", "foo.count")) .build(); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java index a363247ff52..8d5bba77844 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java @@ -19,7 +19,7 @@ import org.junit.Test; import java.util.List; import static ai.vespa.metricsproxy.TestUtil.getFileContents; -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; +import static ai.vespa.metricsproxy.core.VespaMetrics.vespaMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.rpc.IntegrationTester.CUSTOM_CONSUMER_ID; import static ai.vespa.metricsproxy.rpc.IntegrationTester.MONITORING_SYSTEM; @@ -144,7 +144,7 @@ public class RpcMetricsTest { assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").length(), is(1)); if (jsonObject.getJSONObject("metrics").has("foo_count")) { assertThat(jsonObject.getJSONObject("metrics").getInt("foo_count"), is(1)); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(VESPA_CONSUMER_ID.id)); + assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(vespaMetricsConsumerId.id)); } else { assertThat(jsonObject.getJSONObject("metrics").getInt("foo.count"), is(1)); assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(CUSTOM_CONSUMER_ID.id)); @@ -190,7 +190,7 @@ public class RpcMetricsTest { assertNotNull("Did not find expected metric with name 'bar'", m2); try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { - String response = getAllMetricNamesForService(services.get(0).getMonitoringName(), VESPA_CONSUMER_ID, rpcClient); + String response = getAllMetricNamesForService(services.get(0).getMonitoringName(), vespaMetricsConsumerId, rpcClient); assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,")); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java index d86c8745ceb..e00c1dabf0f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java @@ -59,6 +59,7 @@ public class CommandLine { private Duration sigTermGracePeriod = DEFAULT_SIGTERM_GRACE_PERIOD; private Duration sigKillGracePeriod = DEFAULT_SIGKILL_GRACE_PERIOD; private Predicate<Integer> successfulExitCodePredicate = code -> code == 0; + private boolean waitForTermination = true; public CommandLine(TaskContext taskContext, ProcessFactory processFactory) { this.taskContext = taskContext; @@ -242,6 +243,15 @@ public class CommandLine { return this; } + /** + * WARNING: This will leave the child as a zombie process until this process dies. + * I.e. only use this just before or a limited number of times per host admin restart. + */ + public CommandLine doNotWaitForTermination() { + this.waitForTermination = false; + return this; + } + public List<String> getArguments() { return Collections.unmodifiableList(arguments); } // Accessor fields necessary for classes in this package. Could be public if necessary. @@ -256,6 +266,10 @@ public class CommandLine { private CommandResult doExecute() { try (ChildProcess2 child = processFactory.spawn(this)) { + if (!waitForTermination) { + return new CommandResult(this, 0, ""); + } + child.waitForTermination(); int exitCode = child.exitCode(); if (!successfulExitCodePredicate.test(exitCode)) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 4d0c61a4177..ba63376d61e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -20,8 +20,6 @@ import java.util.stream.Collectors; */ public class Autoscaler { - private static final int minimumMeasurementsPerNode = 60; // 1 hour - /** What cost difference factor is worth a reallocation? */ private static final double costDifferenceWorthReallocation = 0.1; /** What difference factor for a resource is worth a reallocation? */ @@ -106,16 +104,22 @@ public class Autoscaler { // Require a total number of measurements scaling with the number of nodes, // but don't require that we have at least that many from every node - if (window.measurementCount()/clusterNodes.size() < minimumMeasurementsPerNode) return Optional.empty(); + if (window.measurementCount()/clusterNodes.size() < minimumMeasurementsPerNode(clusterType)) return Optional.empty(); if (window.hostnames() != clusterNodes.size()) return Optional.empty(); return Optional.of(window.average()); } - /** The duration of the window we need to consider to make a scaling decision */ - private Duration scalingWindow(ClusterSpec.Type clusterType) { - if (clusterType.isContent()) return Duration.ofHours(12); // Ideally we should use observed redistribution time - return Duration.ofHours(12); // TODO: Measure much more often to get this down to minutes. And, ideally we should take node startup time into account + /** The duration of the window we need to consider to make a scaling decision. See also minimumMeasurementsPerNode */ + static Duration scalingWindow(ClusterSpec.Type clusterType) { + if (clusterType.isContent()) return Duration.ofHours(12); + return Duration.ofHours(1); + } + + /** Measurements are currently taken once a minute. See also scalingWindow */ + static int minimumMeasurementsPerNode(ClusterSpec.Type clusterType) { + if (clusterType.isContent()) return 60; + return 20; } public static boolean unstable(List<Node> nodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java index b0d73833bc6..daed5a34873 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java @@ -24,7 +24,7 @@ public interface NodeMetrics { private final String hostname; private final String name; - private long timestampSecond; + private final long timestampSecond; private final double value; public MetricValue(String hostname, String name, long timestampSecond, double value) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java index 7fa81f0b78e..316708732a7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java @@ -1,8 +1,11 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; + import java.time.Clock; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -11,7 +14,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.logging.Logger; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -22,21 +25,34 @@ import java.util.stream.Collectors; */ public class NodeMetricsDb { - private static final Duration dbWindow = Duration.ofHours(24); + private final NodeRepository nodeRepository; /** Measurements by key. Each list of measurements is sorted by increasing timestamp */ - private Map<MeasurementKey, List<Measurement>> db = new HashMap<>(); + private final Map<NodeMeasurementsKey, NodeMeasurements> db = new HashMap<>(); /** Lock all access for now since we modify lists inside a map */ private final Object lock = new Object(); - /** Add a measurement to this */ + public NodeMetricsDb(NodeRepository nodeRepository) { + this.nodeRepository = nodeRepository; + } + + /** Add measurements to this */ public void add(Collection<NodeMetrics.MetricValue> metricValues) { synchronized (lock) { for (var value : metricValues) { Resource resource = Resource.fromMetric(value.name()); - List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(), resource), - (__) -> new ArrayList<>()); + NodeMeasurementsKey key = new NodeMeasurementsKey(value.hostname(), resource); + NodeMeasurements measurements = db.get(key); + if (measurements == null) { // new node + Optional<Node> node = nodeRepository.getNode(value.hostname()); + if (node.isEmpty()) continue; + if (node.get().allocation().isEmpty()) continue; + measurements = new NodeMeasurements(value.hostname(), + resource, + node.get().allocation().get().membership().cluster().type()); + db.put(key, measurements); + } measurements.add(new Measurement(value.timestampSecond() * 1000, (float)resource.valueFromMetric(value.value()))); } @@ -46,17 +62,12 @@ public class NodeMetricsDb { /** Must be called intermittently (as long as add is called) to gc old measurements */ public void gc(Clock clock) { synchronized (lock) { - // TODO: We may need to do something more complicated to avoid spending too much memory to - // lower the measurement interval (see NodeRepositoryMaintenance) // Each measurement is Object + long + float = 16 + 8 + 4 = 28 bytes - // 24 hours with 1k nodes and 3 resources and 1 measurement/sec is about 10Gb - - long oldestTimestamp = clock.instant().minus(dbWindow).toEpochMilli(); - for (Iterator<List<Measurement>> i = db.values().iterator(); i.hasNext(); ) { - List<Measurement> measurements = i.next(); - while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) - measurements.remove(0); + // 12 hours with 1k nodes and 3 resources and 1 measurement/sec is about 5Gb + for (Iterator<NodeMeasurements> i = db.values().iterator(); i.hasNext(); ) { + var measurements = i.next(); + measurements.removeOlderThan(clock.instant().minus(Autoscaler.scalingWindow(measurements.type)).toEpochMilli()); if (measurements.isEmpty()) i.remove(); } @@ -71,19 +82,22 @@ public class NodeMetricsDb { public class Window { private final long startTime; - private List<MeasurementKey> keys; + private final List<NodeMeasurementsKey> keys; private Window(Instant startTime, Resource resource, List<String> hostnames) { this.startTime = startTime.toEpochMilli(); - keys = hostnames.stream().map(hostname -> new MeasurementKey(hostname, resource)).collect(Collectors.toList()); + keys = hostnames.stream().map(hostname -> new NodeMeasurementsKey(hostname, resource)).collect(Collectors.toList()); } public int measurementCount() { synchronized (lock) { - return (int) keys.stream() - .flatMap(key -> db.getOrDefault(key, List.of()).stream()) - .filter(measurement -> measurement.timestamp >= startTime) - .count(); + int count = 0; + for (NodeMeasurementsKey key : keys) { + NodeMeasurements measurements = db.get(key); + if (measurements == null) continue; + count += measurements.after(startTime).size(); + } + return count; } } @@ -91,8 +105,8 @@ public class NodeMetricsDb { public int hostnames() { synchronized (lock) { int count = 0; - for (MeasurementKey key : keys) { - List<Measurement> measurements = db.get(key); + for (NodeMeasurementsKey key : keys) { + NodeMeasurements measurements = db.get(key); if (measurements == null || measurements.isEmpty()) continue; if (measurements.get(measurements.size() - 1).timestamp >= startTime) @@ -106,8 +120,8 @@ public class NodeMetricsDb { synchronized (lock) { double sum = 0; int count = 0; - for (MeasurementKey key : keys) { - List<Measurement> measurements = db.get(key); + for (NodeMeasurementsKey key : keys) { + NodeMeasurements measurements = db.get(key); if (measurements == null) continue; int index = measurements.size() - 1; @@ -124,12 +138,12 @@ public class NodeMetricsDb { } - private static class MeasurementKey { + private static class NodeMeasurementsKey { private final String hostname; private final Resource resource; - public MeasurementKey(String hostname, Resource resource) { + public NodeMeasurementsKey(String hostname, Resource resource) { this.hostname = hostname; this.resource = resource; } @@ -142,15 +156,49 @@ public class NodeMetricsDb { @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof MeasurementKey)) return false; - MeasurementKey other = (MeasurementKey)o; + if ( ! (o instanceof NodeMeasurementsKey)) return false; + NodeMeasurementsKey other = (NodeMeasurementsKey)o; if ( ! this.hostname.equals(other.hostname)) return false; if ( ! this.resource.equals(other.resource)) return false; return true; } @Override - public String toString() { return "measurements of " + resource + " for " + hostname; } + public String toString() { return "key to measurements of " + resource + " for " + hostname; } + + } + + private static class NodeMeasurements { + + private final String hostname; + private final Resource resource; + private final ClusterSpec.Type type; + private final List<Measurement> measurements = new ArrayList<>(); + + public NodeMeasurements(String hostname, Resource resource, ClusterSpec.Type type) { + this.hostname = hostname; + this.resource = resource; + this.type = type; + } + + void removeOlderThan(long oldestTimestamp) { + while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) + measurements.remove(0); + } + + boolean isEmpty() { return measurements.isEmpty(); } + + int size() { return measurements.size(); } + + Measurement get(int index) { return measurements.get(index); } + + void add(Measurement measurement) { measurements.add(measurement); } + + public List<Measurement> after(long oldestTimestamp) { + return measurements.stream() + .filter(measurement -> measurement.timestamp >= oldestTimestamp) + .collect(Collectors.toList()); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java index acdd419c0de..1361faba66c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java @@ -62,7 +62,8 @@ public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics .stream() .findFirst(); if (metricsV2Container.isEmpty()) return Collections.emptyList(); - String url = "http://" + metricsV2Container.get().hostname() + ":" + 4080 + apiPath + "?consumer=default"; + // Consumer 'autoscaling' defined in com.yahoo.vespa.model.admin.monitoring.MetricConsumer + String url = "http://" + metricsV2Container.get().hostname() + ":" + 4080 + apiPath + "?consumer=autoscaling"; String response = httpClient.get(url); return new MetricsResponse(response).metrics(); } @@ -114,7 +115,6 @@ public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics } } - } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index a5482281ef1..0890908dc80 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -182,7 +182,9 @@ public class NodeRepositoryMaintenance extends AbstractComponent { provisionedExpiry = Duration.ofHours(4); rebalancerInterval = Duration.ofMinutes(40); redeployMaintainerInterval = Duration.ofMinutes(1); - reservationExpiry = Duration.ofMinutes(15); // Need to be long enough for deployment to be finished for all config model versions + // Need to be long enough for deployment to be finished for all config model versions + // Should be equal to timeout for deployments + reservationExpiry = zone.system().isCd() ? Duration.ofMinutes(5) : Duration.ofMinutes(30); scalingSuggestionsInterval = Duration.ofMinutes(31); spareCapacityMaintenanceInterval = Duration.ofMinutes(10); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index 3cb0f935888..e3128dfc8e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -7,7 +7,6 @@ import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.LinkedHashSet; @@ -27,15 +26,11 @@ import java.util.stream.Collectors; public class PeriodicApplicationMaintainer extends ApplicationMaintainer { private final Duration minTimeBetweenRedeployments; - private final Clock clock; - private final Instant start; PeriodicApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository, Duration interval, Duration minTimeBetweenRedeployments) { super(deployer, metric, nodeRepository, interval); this.minTimeBetweenRedeployments = minTimeBetweenRedeployments; - this.clock = nodeRepository.clock(); - this.start = clock.instant(); } @Override @@ -51,7 +46,7 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { // Returns the applications that need to be redeployed by this config server at this point in time. @Override protected Set<ApplicationId> applicationsNeedingMaintenance() { - if (waitInitially()) return Set.of(); + if (deployer().bootstrapping()) return Set.of(); // Collect all deployment times before sorting as deployments may happen while we build the set, breaking // the comparable contract. Stale times are fine as the time is rechecked in ApplicationMaintainer#deployWithLock @@ -75,11 +70,6 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { return true; } - // TODO: Do not start deploying until some time has gone (ideally only until bootstrap of config server is finished) - private boolean waitInitially() { - return clock.instant().isBefore(start.plus(minTimeBetweenRedeployments)); - } - protected List<Node> nodesNeedingMaintenance() { return nodeRepository().getNodes(Node.State.active); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 3efd6e417cb..8e3ae6358df 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.status.HostStatus; import java.io.IOException; import java.io.OutputStream; @@ -164,6 +165,10 @@ class NodesResponse extends HttpResponse { orchestrator.apply(new HostName(node.hostname())) .ifPresent(info -> { object.setBool("allowedToBeDown", info.status().isSuspended()); + // TODO: Remove allowedToBeDown as a special-case of orchestratorHostStatus + if (info.status() != HostStatus.NO_REMARKS) { + object.setString("orchestratorStatus", info.status().asString()); + } info.suspendedSince().ifPresent(since -> object.setLong("suspendedSinceMillis", since.toEpochMilli())); }); }); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index ae3d6ebf815..06ece5b3767 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import java.time.Clock; @@ -44,6 +43,7 @@ public class MockDeployer implements Deployer { private final ReentrantLock lock = new ReentrantLock(); private boolean failActivate = false; + private boolean bootstrapping = true; /** Create a mock deployer which returns empty on every deploy request. */ @Inject @@ -83,6 +83,8 @@ public class MockDeployer implements Deployer { public void setFailActivate(boolean failActivate) { this.failActivate = failActivate; } + public void setBootstrapping(boolean bootstrapping) { this.bootstrapping = bootstrapping; } + @Override public Optional<Deployment> deployFromLocalActive(ApplicationId id, boolean bootstrap) { return deployFromLocalActive(id, Duration.ofSeconds(60)); @@ -117,6 +119,11 @@ public class MockDeployer implements Deployer { return Optional.ofNullable(lastDeployTimes.get(application)); } + @Override + public boolean bootstrapping() { + return bootstrapping; + } + public void removeApplication(ApplicationId applicationId) { new MockDeployment(provisioner, new ApplicationContext(applicationId, List.of())).activate(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index d7829df4152..511d2f988d0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -34,9 +34,9 @@ public class NodeRepositoryTest { NodeRepositoryTester tester = new NodeRepositoryTester(); assertEquals(0, tester.nodeRepository().getNodes().size()); - tester.addNode("id1", "host1", "default", NodeType.host); - tester.addNode("id2", "host2", "default", NodeType.host); - tester.addNode("id3", "host3", "default", NodeType.host); + tester.addHost("id1", "host1", "default", NodeType.host); + tester.addHost("id2", "host2", "default", NodeType.host); + tester.addHost("id3", "host3", "default", NodeType.host); assertEquals(3, tester.nodeRepository().getNodes().size()); @@ -50,7 +50,7 @@ public class NodeRepositoryTest { @Test public void only_allow_docker_containers_remove_in_ready() { NodeRepositoryTester tester = new NodeRepositoryTester(); - tester.addNode("id1", "host1", "docker", NodeType.tenant); + tester.addHost("id1", "host1", "docker", NodeType.tenant); try { tester.nodeRepository().removeRecursively("host1"); // host1 is in state provisioned @@ -66,9 +66,9 @@ public class NodeRepositoryTest { @Test public void only_remove_tenant_docker_containers_for_new_allocations() { NodeRepositoryTester tester = new NodeRepositoryTester(); - tester.addNode("host1", "host1", "default", NodeType.tenant); - tester.addNode("host2", "host2", "docker", NodeType.tenant); - tester.addNode("cfg1", "cfg1", "docker", NodeType.config); + tester.addHost("host1", "host1", "default", NodeType.tenant); + tester.addHost("host2", "host2", "docker", NodeType.tenant); + tester.addHost("cfg1", "cfg1", "docker", NodeType.config); tester.setNodeState("host1", Node.State.dirty); tester.setNodeState("host2", Node.State.dirty); @@ -87,8 +87,8 @@ public class NodeRepositoryTest { @Test public void fail_readying_with_hard_fail() { NodeRepositoryTester tester = new NodeRepositoryTester(); - tester.addNode("host1", "host1", "default", NodeType.tenant); - tester.addNode("host2", "host2", "default", NodeType.tenant); + tester.addHost("host1", "host1", "default", NodeType.tenant); + tester.addHost("host2", "host2", "default", NodeType.tenant); tester.setNodeState("host1", Node.State.dirty); tester.setNodeState("host2", Node.State.dirty); @@ -113,8 +113,8 @@ public class NodeRepositoryTest { public void delete_host_only_after_all_the_children_have_been_deleted() { NodeRepositoryTester tester = new NodeRepositoryTester(); - tester.addNode("id1", "host1", "default", NodeType.host); - tester.addNode("id2", "host2", "default", NodeType.host); + tester.addHost("id1", "host1", "default", NodeType.host); + tester.addHost("id2", "host2", "default", NodeType.host); tester.addNode("node10", "node10", "host1", "docker", NodeType.tenant); tester.addNode("node11", "node11", "host1", "docker", NodeType.tenant); tester.addNode("node12", "node12", "host1", "docker", NodeType.tenant); @@ -151,7 +151,7 @@ public class NodeRepositoryTest { String cfghost1 = "cfghost1"; String cfg1 = "cfg1"; - tester.addNode("id1", cfghost1, "default", NodeType.confighost); + tester.addHost("id1", cfghost1, "default", NodeType.confighost); tester.addNode("id2", cfg1, cfghost1, "docker", NodeType.config); tester.setNodeState(cfghost1, Node.State.active); tester.setNodeState(cfg1, Node.State.active); @@ -177,8 +177,8 @@ public class NodeRepositoryTest { Instant testStart = tester.nodeRepository().clock().instant(); tester.clock().advance(Duration.ofSeconds(1)); - tester.addNode("id1", "host1", "default", NodeType.host); - tester.addNode("id2", "host2", "default", NodeType.host); + tester.addHost("id1", "host1", "default", NodeType.host); + tester.addHost("id2", "host2", "default", NodeType.host); assertFalse(tester.nodeRepository().getNode("host1").get().history().hasEventAfter(History.Event.Type.deprovisioned, testStart)); // Set host 1 properties and deprovision it @@ -196,7 +196,7 @@ public class NodeRepositoryTest { assertTrue(host1.history().hasEventAfter(History.Event.Type.deprovisioned, testStart)); // Adding it again preserves some information from the deprovisioned host and removes it - tester.addNode("id2", "host1", "default", NodeType.host); + tester.addHost("id2", "host1", "default", NodeType.host); host1 = tester.nodeRepository().getNode("host1").get(); assertEquals("This is the newly added node", "id2", host1.id()); assertFalse("The old 'host1' is removed", @@ -213,8 +213,8 @@ public class NodeRepositoryTest { public void dirty_host_only_if_we_can_dirty_children() { NodeRepositoryTester tester = new NodeRepositoryTester(); - tester.addNode("id1", "host1", "default", NodeType.host); - tester.addNode("id2", "host2", "default", NodeType.host); + tester.addHost("id1", "host1", "default", NodeType.host); + tester.addHost("id2", "host2", "default", NodeType.host); tester.addNode("node10", "node10", "host1", "docker", NodeType.tenant); tester.addNode("node11", "node11", "host1", "docker", NodeType.tenant); tester.addNode("node12", "node12", "host1", "docker", NodeType.tenant); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 9b0500303d8..92f7e7404a6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; @@ -54,7 +55,7 @@ public class NodeRepositoryTester { return nodeRepository.getNodes(type, inState); } - public Node addNode(String id, String hostname, String flavor, NodeType type) { + public Node addHost(String id, String hostname, String flavor, NodeType type) { Node node = nodeRepository.createNode(id, hostname, Optional.empty(), nodeFlavors.getFlavorOrThrow(flavor), type); return nodeRepository.addNodes(Collections.singletonList(node), Agent.system).get(0); @@ -66,6 +67,12 @@ public class NodeRepositoryTester { return nodeRepository.addNodes(Collections.singletonList(node), Agent.system).get(0); } + public Node addNode(String id, String hostname, String parentHostname, NodeResources resources) { + Node node = nodeRepository.createNode(id, hostname, Optional.of(parentHostname), + new Flavor(resources), NodeType.tenant); + return nodeRepository.addNodes(Collections.singletonList(node), Agent.system).get(0); + } + /** * Moves a node directly to the given state without doing any validation, useful * to create wanted test scenario without having to move every node through series diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index cb39e8fecce..0a127eacae1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -67,7 +67,7 @@ class AutoscalingTester { .build(); hostResourcesCalculator = new MockHostResourcesCalculator(zone); - db = new NodeMetricsDb(); + db = new NodeMetricsDb(provisioningTester.nodeRepository()); autoscaler = new Autoscaler(db, nodeRepository()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java index 0a3436bc379..976aeb2346a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java @@ -1,7 +1,16 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.time.Duration; @@ -14,23 +23,33 @@ public class NodeMetricsDbTest { @Test public void testNodeMetricsDb() { - ManualClock clock = new ManualClock(); - NodeMetricsDb db = new NodeMetricsDb(); + ProvisioningTester tester = new ProvisioningTester.Builder().build(); + tester.makeReadyHosts(10, new NodeResources(10, 100, 1000, 10)).deployZoneApp(); + ApplicationId app1 = tester.makeApplicationId("app1"); + tester.deployZoneApp(); + var hosts = + tester.activate(app1, + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("7.0").build(), + Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 1)))); + String node0 = hosts.iterator().next().hostname(); + + ManualClock clock = tester.clock(); + NodeMetricsDb db = new NodeMetricsDb(tester.nodeRepository()); List<NodeMetrics.MetricValue> values = new ArrayList<>(); for (int i = 0; i < 40; i++) { - values.add(new NodeMetrics.MetricValue("host0", "cpu.util", clock.instant().getEpochSecond(), 0.9f)); - clock.advance(Duration.ofHours(1)); + values.add(new NodeMetrics.MetricValue(node0, "cpu.util", clock.instant().getEpochSecond(), 0.9f)); + clock.advance(Duration.ofMinutes(10)); } db.add(values); // Avoid off-by-one bug when the below windows starts exactly on one of the above getEpochSecond() timestamps. clock.advance(Duration.ofMinutes(1)); - assertEquals(29, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); - assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount()); + assertEquals(35, db.getWindow(clock.instant().minus(Duration.ofHours(6)), Resource.cpu, List.of(node0)).measurementCount()); + assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(6)), Resource.memory, List.of(node0)).measurementCount()); db.gc(clock); - assertEquals(23, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu, List.of("host0")).measurementCount()); - assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount()); + assertEquals( 5, db.getWindow(clock.instant().minus(Duration.ofHours(6)), Resource.cpu, List.of(node0)).measurementCount()); + assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(6)), Resource.memory, List.of(node0)).measurementCount()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java index 6bf52218302..d418d818ef3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java @@ -41,7 +41,7 @@ public class NodeMetricsFetcherTest { { httpClient.cannedResponse = cannedResponseForApplication1; List<NodeMetrics.MetricValue> values = new ArrayList<>(fetcher.fetchMetrics(application1)); - assertEquals("http://host-1.yahoo.com:4080/metrics/v2/values?consumer=default", + assertEquals("http://host-1.yahoo.com:4080/metrics/v2/values?consumer=autoscaling", httpClient.requestsReceived.get(0)); assertEquals(5, values.size()); assertEquals("metric value cpu.util: 16.2 at 1970-01-01T00:20:34Z for host-1.yahoo.com", values.get(0).toString()); @@ -54,7 +54,7 @@ public class NodeMetricsFetcherTest { { httpClient.cannedResponse = cannedResponseForApplication2; List<NodeMetrics.MetricValue> values = new ArrayList<>(fetcher.fetchMetrics(application2)); - assertEquals("http://host-3.yahoo.com:4080/metrics/v2/values?consumer=default", + assertEquals("http://host-3.yahoo.com:4080/metrics/v2/values?consumer=autoscaling", httpClient.requestsReceived.get(1)); assertEquals(3, values.size()); assertEquals("metric value cpu.util: 10.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(0).toString()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index c34db3210c1..88195cf0ed9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -54,7 +54,7 @@ public class AutoscalingMaintainerTest { app2, new MockDeployer.ApplicationContext(app2, cluster2, Capacity.from(new ClusterResources(2, 1, highResources)))); MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); - NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); + NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(tester.nodeRepository()); AutoscalingMaintainer maintainer = new AutoscalingMaintainer(tester.nodeRepository(), nodeMetricsDb, deployer, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 099282205fe..ec830a7dc31 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -87,8 +87,9 @@ public class PeriodicApplicationMaintainerTest { // Create applications fixture.activate(); - // Exhaust initial wait period + // Exhaust initial wait period and set bootstrapping to be done clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + fixture.setBootstrapping(false); // Fail and park some nodes nodeRepository.fail(nodeRepository.getNodes(fixture.app1).get(3).hostname(), Agent.system, "Failing to unit test"); @@ -161,9 +162,15 @@ public class PeriodicApplicationMaintainerTest { // Exhaust initial wait period clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); - // First deployment of applications + // Will not do any deployments, as bootstrapping is still in progress + fixture.runApplicationMaintainer(); + assertEquals("No deployment expected", 2, fixture.deployer.redeployments); + + // First deployment of applications will happen now, as bootstrapping is done + fixture.setBootstrapping(false); fixture.runApplicationMaintainer(); assertEquals("No deployment expected", 4, fixture.deployer.redeployments); + Instant firstDeployTime = clock.instant(); assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get()); assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get()); @@ -186,8 +193,9 @@ public class PeriodicApplicationMaintainerTest { public void queues_all_eligible_applications_for_deployment() throws Exception { fixture.activate(); - // Exhaust initial wait period + // Exhaust initial wait period and set bootstrapping to be done clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + fixture.setBootstrapping(false); // Lock deployer to simulate slow deployments fixture.deployer.lock().lockInterruptibly(); @@ -298,6 +306,10 @@ public class PeriodicApplicationMaintainerTest { return NodeList.copyOf(nodeRepository.getNodes(NodeType.tenant, states)); } + void setBootstrapping(boolean bootstrapping) { + deployer.setBootstrapping(bootstrapping); + } + } private static class TestablePeriodicApplicationMaintainer extends PeriodicApplicationMaintainer { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index be5c7f423c7..31a9fcb8999 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -44,7 +44,7 @@ public class ScalingSuggestionsMaintainerTest { ApplicationId app2 = tester.makeApplicationId("app2"); ClusterSpec cluster2 = tester.contentClusterSpec(); - NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); + NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(tester.nodeRepository()); tester.makeReadyNodes(20, "flt", NodeType.host, 8); tester.deployZoneApp(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index e45ea09d372..edbbb3ed2e3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -136,7 +136,7 @@ public class InfraDeployerImplTest { } private Node addNode(int id, Node.State state, Optional<Version> wantedVespaVersion) { - Node node = tester.addNode("id-" + id, "node-" + id, "default", nodeType); + Node node = tester.addHost("id-" + id, "node-" + id, "default", nodeType); Optional<Node> nodeWithAllocation = wantedVespaVersion.map(version -> { ClusterSpec clusterSpec = application.getClusterSpecWithVersion(version).with(Optional.of(ClusterSpec.Group.from(0))); ClusterMembership membership = ClusterMembership.from(clusterSpec, 1); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json index af3552945d9..7ca5048ac3b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json @@ -28,6 +28,7 @@ "wantedVespaVersion": "6.42.0", "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, "allowedToBeDown": true, + "orchestratorStatus": "ALLOWED_TO_BE_DOWN", "suspendedSinceMillis": 0, "rebootGeneration": 3, "currentRebootGeneration": 1, diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index baa98790d28..333edd4b3f9 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; import com.yahoo.vespa.orchestrator.policy.Policy; +import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostInfo; @@ -76,13 +77,13 @@ public class OrchestratorImpl implements Orchestrator { { this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, - new ApplicationApiFactory(configServerConfig.zookeeperserver().size())), + new ApplicationApiFactory(configServerConfig.zookeeperserver().size(), Clock.systemUTC())), clusterControllerClientFactory, statusService, serviceMonitor, orchestratorConfig.serviceMonitorConvergenceLatencySeconds(), Clock.systemUTC(), - new ApplicationApiFactory(configServerConfig.zookeeperserver().size()), + new ApplicationApiFactory(configServerConfig.zookeeperserver().size(), Clock.systemUTC()), flagSource); } @@ -227,6 +228,7 @@ public class OrchestratorImpl implements Orchestrator { void suspendGroup(OrchestratorContext context, NodeGroup nodeGroup) throws HostStateChangeDeniedException { ApplicationInstanceReference applicationReference = nodeGroup.getApplicationReference(); + final SuspensionReasons suspensionReasons; try (ApplicationLock lock = statusService.lockApplication(context, applicationReference)) { ApplicationInstanceStatus appStatus = lock.getApplicationInstanceStatus(); if (appStatus == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { @@ -235,8 +237,10 @@ public class OrchestratorImpl implements Orchestrator { ApplicationApi applicationApi = applicationApiFactory.create( nodeGroup, lock, clusterControllerClientFactory); - policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi); + suspensionReasons = policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi); } + + suspensionReasons.makeLogMessage().ifPresent(log::info); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java index fb52eed2048..d913a1791ba 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java @@ -4,21 +4,26 @@ package com.yahoo.vespa.orchestrator.model; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.status.ApplicationLock; +import java.time.Clock; + /** * @author mpolden */ public class ApplicationApiFactory { private final int numberOfConfigServers; + private final Clock clock; - public ApplicationApiFactory(int numberOfConfigServers) { + public ApplicationApiFactory(int numberOfConfigServers, Clock clock) { this.numberOfConfigServers = numberOfConfigServers; + this.clock = clock; } public ApplicationApi create(NodeGroup nodeGroup, ApplicationLock lock, ClusterControllerClientFactory clusterControllerClientFactory) { - return new ApplicationApiImpl(nodeGroup, lock, clusterControllerClientFactory, numberOfConfigServers); + return new ApplicationApiImpl(nodeGroup, lock, clusterControllerClientFactory, + numberOfConfigServers, clock); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index ccc89fa9191..7ee1395b7e7 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.ApplicationLock; +import java.time.Clock; import java.util.Collection; import java.util.Comparator; import java.util.HashSet; @@ -33,16 +34,18 @@ public class ApplicationApiImpl implements ApplicationApi { private final ApplicationInstance applicationInstance; private final NodeGroup nodeGroup; private final ApplicationLock lock; + private final Clock clock; private final List<ClusterApi> clusterInOrder; private final HostInfos hostInfos; public ApplicationApiImpl(NodeGroup nodeGroup, ApplicationLock lock, ClusterControllerClientFactory clusterControllerClientFactory, - int numberOfConfigServers) { + int numberOfConfigServers, Clock clock) { this.applicationInstance = nodeGroup.getApplication(); this.nodeGroup = nodeGroup; this.lock = lock; + this.clock = clock; Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance); this.hostInfos = lock.getHostInfos(); this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, numberOfConfigServers); @@ -122,7 +125,8 @@ public class ApplicationApiImpl implements ApplicationApi { nodeGroup, hostInfos, clusterControllerClientFactory, - numberOfConfigServers)) + numberOfConfigServers, + clock)) .sorted(ApplicationApiImpl::compareClusters) .collect(Collectors.toList()); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java index 65c45c8df76..2e7a63ddb2f 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator.model; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import java.util.Optional; @@ -17,7 +18,8 @@ public interface ClusterApi { ServiceType serviceType(); boolean isStorageCluster(); - boolean noServicesInGroupIsUp(); + /** Returns the reasons no services are up in the implied group, or empty if some services are up. */ + Optional<SuspensionReasons> reasonsForNoServicesInGroupIsUp(); boolean noServicesOutsideGroupIsDown(); int percentageOfServicesDown(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index ae3136543a7..7417318c572 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -8,9 +8,13 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -24,12 +28,14 @@ import java.util.stream.Stream; * @author hakonhall */ class ClusterApiImpl implements ClusterApi { + static final Duration downMoratorium = Duration.ofSeconds(30); private final ApplicationApi applicationApi; private final ServiceCluster serviceCluster; private final NodeGroup nodeGroup; private final HostInfos hostInfos; private final ClusterControllerClientFactory clusterControllerClientFactory; + private final Clock clock; private final Set<ServiceInstance> servicesInGroup; private final Set<ServiceInstance> servicesDownInGroup; private final Set<ServiceInstance> servicesNotInGroup; @@ -53,12 +59,14 @@ class ClusterApiImpl implements ClusterApi { NodeGroup nodeGroup, HostInfos hostInfos, ClusterControllerClientFactory clusterControllerClientFactory, - int numberOfConfigServers) { + int numberOfConfigServers, + Clock clock) { this.applicationApi = applicationApi; this.serviceCluster = serviceCluster; this.nodeGroup = nodeGroup; this.hostInfos = hostInfos; this.clusterControllerClientFactory = clusterControllerClientFactory; + this.clock = clock; Map<Boolean, Set<ServiceInstance>> serviceInstancesByLocality = serviceCluster.serviceInstances().stream() @@ -73,7 +81,7 @@ class ClusterApiImpl implements ClusterApi { servicesDownAndNotInGroup = servicesNotInGroup.stream().filter(this::serviceEffectivelyDown).collect(Collectors.toSet()); int serviceInstances = serviceCluster.serviceInstances().size(); - if (serviceCluster.isConfigServerClusterLike() && serviceInstances < numberOfConfigServers) { + if (serviceCluster.isConfigServerLike() && serviceInstances < numberOfConfigServers) { missingServices = numberOfConfigServers - serviceInstances; descriptionOfMissingServices = missingServices + " missing config server" + (missingServices > 1 ? "s" : ""); } else { @@ -108,8 +116,36 @@ class ClusterApiImpl implements ClusterApi { } @Override - public boolean noServicesInGroupIsUp() { - return servicesDownInGroup.size() == servicesInGroup.size(); + public Optional<SuspensionReasons> reasonsForNoServicesInGroupIsUp() { + SuspensionReasons reasons = new SuspensionReasons(); + + for (ServiceInstance service : servicesInGroup) { + if (hostStatus(service.hostName()).isSuspended()) { + reasons.mergeWith(SuspensionReasons.nothingNoteworthy()); + continue; + } + + if (service.serviceStatus() == ServiceStatus.DOWN) { + Optional<Instant> since = service.serviceStatusInfo().since(); + if (since.isEmpty()) { + reasons.mergeWith(SuspensionReasons.isDown(service)); + continue; + } + + // Make sure services truly are down for some period of time before we allow suspension. + // On the other hand, a service coming down and up repeatedly should probably + // also be allowed... difficult without keeping track of history in a better way. + final Duration downDuration = Duration.between(since.get(), clock.instant()); + if (downDuration.compareTo(downMoratorium) > 0) { + reasons.mergeWith(SuspensionReasons.downSince(service, since.get(), downDuration)); + continue; + } + } + + return Optional.empty(); + } + + return Optional.of(reasons); } int missingServices() { return missingServices; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java index 7c2daf0b597..a0738b725ec 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterPolicy.java @@ -8,8 +8,10 @@ public interface ClusterPolicy { * There's an implicit group of nodes known to clusterApi. This method answers whether * it would be fine, just looking at this cluster (and disregarding Cluster Controller/storage * which is handled separately), to allow all services on all the nodes in the group to go down. + * + * @return notable reasons for why this group is fine with going down. */ - void verifyGroupGoingDownIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException; + SuspensionReasons verifyGroupGoingDownIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException; /** * There's an implicit group of nodes known to clusterApi. This method answers whether diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index ccb0bb57186..4e11961cb1b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -4,23 +4,26 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.vespa.orchestrator.model.ClusterApi; import com.yahoo.vespa.orchestrator.model.VespaModelUtil; +import java.util.Optional; + import static com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy.ENOUGH_SERVICES_UP_CONSTRAINT; public class HostedVespaClusterPolicy implements ClusterPolicy { @Override - public void verifyGroupGoingDownIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException { + public SuspensionReasons verifyGroupGoingDownIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException { if (clusterApi.noServicesOutsideGroupIsDown()) { - return; - } - - if (clusterApi.noServicesInGroupIsUp()) { - return; + return SuspensionReasons.nothingNoteworthy(); } int percentageOfServicesAllowedToBeDown = getConcurrentSuspensionLimit(clusterApi).asPercentage(); if (clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() <= percentageOfServicesAllowedToBeDown) { - return; + return SuspensionReasons.nothingNoteworthy(); + } + + Optional<SuspensionReasons> suspensionReasons = clusterApi.reasonsForNoServicesInGroupIsUp(); + if (suspensionReasons.isPresent()) { + return suspensionReasons.get(); } throw new HostStateChangeDeniedException( @@ -34,7 +37,8 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { } @Override - public void verifyGroupGoingDownPermanentlyIsFine(ClusterApi clusterApi) throws HostStateChangeDeniedException { + public void verifyGroupGoingDownPermanentlyIsFine(ClusterApi clusterApi) + throws HostStateChangeDeniedException { // This policy is similar to verifyGroupGoingDownIsFine, except that services being down in the group // is no excuse to allow suspension (like it is for verifyGroupGoingDownIsFine), since if we grant // suspension in this case they will permanently be down/removed. diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index 8b74d8a40ef..f9b914e391e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -39,11 +39,13 @@ public class HostedVespaPolicy implements Policy { } @Override - public void grantSuspensionRequest(OrchestratorContext context, ApplicationApi application) + public SuspensionReasons grantSuspensionRequest(OrchestratorContext context, ApplicationApi application) throws HostStateChangeDeniedException { + var suspensionReasons = new SuspensionReasons(); + // Apply per-cluster policy for (ClusterApi cluster : application.getClusters()) { - clusterPolicy.verifyGroupGoingDownIsFine(cluster); + suspensionReasons.mergeWith(clusterPolicy.verifyGroupGoingDownIsFine(cluster)); } // Ask Cluster Controller to set UP storage nodes in maintenance. @@ -56,6 +58,8 @@ public class HostedVespaPolicy implements Policy { for (HostName hostName : application.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)) { application.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN); } + + return suspensionReasons; } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java index c410cda23a8..86538450782 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java @@ -15,7 +15,7 @@ public interface Policy { /** * Decide whether to grant a request for temporarily suspending the services on all hosts in the group. */ - void grantSuspensionRequest(OrchestratorContext context, ApplicationApi applicationApi) throws HostStateChangeDeniedException; + SuspensionReasons grantSuspensionRequest(OrchestratorContext context, ApplicationApi applicationApi) throws HostStateChangeDeniedException; void releaseSuspensionGrant(OrchestratorContext context, ApplicationApi application) throws HostStateChangeDeniedException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasons.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasons.java new file mode 100644 index 00000000000..c043396497b --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasons.java @@ -0,0 +1,83 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Information worth logging when/if suspending a host. + * + * @author hakon + */ +public class SuspensionReasons { + private final Map<HostName, List<String>> reasons = new HashMap<>(); + + public static SuspensionReasons nothingNoteworthy() { return new SuspensionReasons(); } + public static SuspensionReasons isDown(ServiceInstance service) { + return new SuspensionReasons().addReason( + service.hostName(), + service.descriptiveName() + " is down"); + } + + public static SuspensionReasons downSince(ServiceInstance service, Instant instant, Duration downDuration) { + return new SuspensionReasons().addReason( + service.hostName(), + service.descriptiveName() + " has been down since " + + // Round to whole second + Instant.ofEpochSecond(instant.getEpochSecond()).toString() + + " (" + downDuration.getSeconds() + " seconds)"); + } + + public SuspensionReasons() {} + + /** An ordered list of all messages, typically useful for testing. */ + public List<String> getMessagesInOrder() { + return reasons.values().stream().flatMap(Collection::stream).sorted().collect(Collectors.toList()); + } + + public SuspensionReasons mergeWith(SuspensionReasons that) { + for (var entry : that.reasons.entrySet()) { + for (var reason : entry.getValue()) { + addReason(entry.getKey(), reason); + } + } + + return this; + } + + /** + * Makes a log message, if there is anything worth logging about the decision to allow suspension, + * that can be directly fed to {@link java.util.logging.Logger#info(String) Logger.info(String)}. + */ + public Optional<String> makeLogMessage() { + if (reasons.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(reasons.entrySet().stream() + .map(entry -> entry.getKey().s() + " suspended because " + String.join(", ", entry.getValue())) + .sorted() + .collect(Collectors.joining("; "))); + } + + /** Package-private for testing. */ + SuspensionReasons addReason(HostName hostname, String message) { + reasons.computeIfAbsent(hostname, h -> new ArrayList<>()).add(message); + return this; + } + + @Override + public String toString() { + return makeLogMessage().orElse(""); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index b32a6aafa56..3580b305c2a 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -73,7 +74,8 @@ import static org.mockito.internal.verification.VerificationModeFactory.atLeastO */ public class OrchestratorImplTest { - private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3); + private final ManualClock clock = new ManualClock(); + private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, clock); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final MockCurator curator = new MockCurator(); private ZkStatusService statusService = new ZkStatusService( @@ -344,6 +346,7 @@ public class OrchestratorImplTest { var applicationApiFactory = mock(ApplicationApiFactory.class); var lock = mock(ApplicationLock.class); + when(policy.grantSuspensionRequest(any(), any())).thenReturn(SuspensionReasons.nothingNoteworthy()); when(serviceMonitor.getApplication(any(HostName.class))).thenReturn(Optional.of(applicationInstance)); when(applicationInstance.reference()).thenReturn(applicationInstanceReference); when(zookeeperStatusService.lockApplication(any(), any())).thenReturn(lock); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index a5cb5cfa630..14d35902738 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.model; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ClusterId; @@ -8,15 +9,19 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; +import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import org.junit.Test; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -43,6 +48,7 @@ public class ClusterApiImplTest { private final ApplicationApi applicationApi = mock(ApplicationApi.class); private final ModelTestUtils modelUtils = new ModelTestUtils(); + private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(1600436659)); @Test public void testServicesDownAndNotInGroup() { @@ -76,7 +82,7 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), hostName5), modelUtils.getHostInfos(), - modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); + modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, clock); assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); assertFalse(clusterApi.isStorageCluster()); @@ -139,16 +145,30 @@ public class ClusterApiImplTest { HostName hostName3 = new HostName("host3"); HostName hostName4 = new HostName("host4"); HostName hostName5 = new HostName("host5"); + HostName hostName6 = new HostName("host6"); + + ServiceInstance service2 = modelUtils.createServiceInstance("service-2", hostName2, ServiceStatus.DOWN); + + // Within down moratorium + Instant downSince5 = clock.instant().minus(ClusterApiImpl.downMoratorium).plus(Duration.ofSeconds(5)); + ServiceInstance service5 = modelUtils.createServiceInstance("service-5", hostName5, + new ServiceStatusInfo(ServiceStatus.DOWN, downSince5, downSince5, Optional.empty(), Optional.empty())); + + // After down moratorium + Instant downSince6 = clock.instant().minus(ClusterApiImpl.downMoratorium).minus(Duration.ofSeconds(5)); + ServiceInstance service6 = modelUtils.createServiceInstance("service-6", hostName6, + new ServiceStatusInfo(ServiceStatus.DOWN, downSince6, downSince6, Optional.empty(), Optional.empty())); ServiceCluster serviceCluster = modelUtils.createServiceCluster( "cluster", new ServiceType("service-type"), Arrays.asList( modelUtils.createServiceInstance("service-1", hostName1, ServiceStatus.UP), - modelUtils.createServiceInstance("service-2", hostName2, ServiceStatus.DOWN), + service2, modelUtils.createServiceInstance("service-3", hostName3, ServiceStatus.UP), modelUtils.createServiceInstance("service-4", hostName4, ServiceStatus.DOWN), - modelUtils.createServiceInstance("service-5", hostName5, ServiceStatus.UP) + service5, + service6 ) ); modelUtils.createApplicationInstance(Collections.singletonList(serviceCluster)); @@ -158,21 +178,33 @@ public class ClusterApiImplTest { modelUtils.createNode(hostName3, HostStatus.ALLOWED_TO_BE_DOWN); modelUtils.createNode(hostName4, HostStatus.ALLOWED_TO_BE_DOWN); modelUtils.createNode(hostName5, HostStatus.NO_REMARKS); - - verifyNoServices(serviceCluster, false, false, hostName1); - verifyNoServices(serviceCluster, true, false, hostName2); - verifyNoServices(serviceCluster, true, false, hostName3); - verifyNoServices(serviceCluster, true, false, hostName4); - verifyNoServices(serviceCluster, false, false, hostName5); - - verifyNoServices(serviceCluster, false, false, hostName1, hostName2); - verifyNoServices(serviceCluster, true, false, hostName2, hostName3); - verifyNoServices(serviceCluster, true, true, hostName2, hostName3, hostName4); - verifyNoServices(serviceCluster, false, true, hostName1, hostName2, hostName3, hostName4); + modelUtils.createNode(hostName6, HostStatus.NO_REMARKS); + + var reason2 = SuspensionReasons.isDown(service2); + var reason6 = SuspensionReasons.downSince(service6, downSince6, Duration.ofSeconds(35)); + var reasons2and6 = new SuspensionReasons().mergeWith(reason2).mergeWith(reason6); + + verifyNoServices(serviceCluster, Optional.empty(), false, hostName1); + verifyNoServices(serviceCluster, Optional.of(reason2), false, hostName2); + verifyNoServices(serviceCluster, Optional.of(SuspensionReasons.nothingNoteworthy()), false, hostName3); + verifyNoServices(serviceCluster, Optional.of(SuspensionReasons.nothingNoteworthy()), false, hostName4); + verifyNoServices(serviceCluster, Optional.empty(), false, hostName5); + verifyNoServices(serviceCluster, Optional.of(reason6), false, hostName6); + + verifyNoServices(serviceCluster, Optional.empty(), false, hostName1, hostName2); + verifyNoServices(serviceCluster, Optional.of(reasons2and6), false, hostName2, hostName3, hostName6); + verifyNoServices(serviceCluster, Optional.of(reasons2and6), false, + hostName2, hostName3, hostName4, hostName6); + verifyNoServices(serviceCluster, Optional.empty(), true, + hostName2, hostName3, hostName4, hostName5, hostName6); + verifyNoServices(serviceCluster, Optional.empty(), false, + hostName1, hostName2, hostName3, hostName4, hostName6); + verifyNoServices(serviceCluster, Optional.empty(), true, + hostName1, hostName2, hostName3, hostName4, hostName5, hostName6); } private void verifyNoServices(ServiceCluster serviceCluster, - boolean expectedNoServicesInGroupIsUp, + Optional<SuspensionReasons> expectedNoServicesInGroupIsUp, boolean expectedNoServicesOutsideGroupIsDown, HostName... groupNodes) { ClusterApiImpl clusterApi = new ClusterApiImpl( @@ -180,9 +212,10 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), groupNodes), modelUtils.getHostInfos(), - modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); + modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, clock); - assertEquals(expectedNoServicesInGroupIsUp, clusterApi.noServicesInGroupIsUp()); + assertEquals(expectedNoServicesInGroupIsUp.map(SuspensionReasons::getMessagesInOrder), + clusterApi.reasonsForNoServicesInGroupIsUp().map(SuspensionReasons::getMessagesInOrder)); assertEquals(expectedNoServicesOutsideGroupIsDown, clusterApi.noServicesOutsideGroupIsDown()); } @@ -210,7 +243,7 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(applicationInstance, hostName1, hostName3), new HostInfos(), - modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); + modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS, clock); assertTrue(clusterApi.isStorageCluster()); assertEquals(Optional.of(hostName1), clusterApi.storageNodeInGroup().map(storageNode -> storageNode.hostName())); @@ -233,6 +266,9 @@ public class ClusterApiImplTest { ServiceType.CONFIG_SERVER, instances ); + for (var instance : instances) { + instance.setServiceCluster(serviceCluster); + } Set<ServiceCluster> serviceClusterSet = Set.of(serviceCluster); @@ -250,7 +286,7 @@ public class ClusterApiImplTest { serviceCluster, new NodeGroup(application, hostnames.get(0)), modelUtils.getHostInfos(), - modelUtils.getClusterControllerClientFactory(), clusterSize); + modelUtils.getClusterControllerClientFactory(), clusterSize, clock); assertEquals(clusterSize - serviceStatusList.size(), clusterApi.missingServices()); assertEquals(clusterSize == serviceStatusList.size(), clusterApi.noServicesOutsideGroupIsDown()); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index 8162542e540..c4087daeb94 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; +import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; @@ -72,9 +73,10 @@ class ModelTestUtils { new ManualClock(), applicationApiFactory(), flagSource); + private final ManualClock clock = new ManualClock(); ApplicationApiFactory applicationApiFactory() { - return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS); + return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS, clock); } HostInfos getHostInfos() { @@ -142,21 +144,19 @@ class ModelTestUtils { ServiceType serviceType, List<ServiceInstance> serviceInstances) { Set<ServiceInstance> serviceInstanceSet = new HashSet<>(serviceInstances); + var cluster = new ServiceCluster(new ClusterId(clusterId), serviceType, serviceInstanceSet); + for (var service : serviceInstanceSet) { + service.setServiceCluster(cluster); + } + return cluster; + } - return new ServiceCluster( - new ClusterId(clusterId), - serviceType, - serviceInstanceSet); + ServiceInstance createServiceInstance(String configId, HostName hostName, ServiceStatus status) { + return new ServiceInstance(new ConfigId(configId), hostName, status); } - ServiceInstance createServiceInstance( - String configId, - HostName hostName, - ServiceStatus status) { - return new ServiceInstance( - new ConfigId(configId), - hostName, - status); + ServiceInstance createServiceInstance(String configId, HostName hostName, ServiceStatusInfo statusInfo) { + return new ServiceInstance(new ConfigId(configId), hostName, statusInfo); } ClusterControllerClientFactory getClusterControllerClientFactory() { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index 4462e886d1b..e5be87ba043 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ClusterApi; @@ -11,6 +12,8 @@ import com.yahoo.vespa.orchestrator.model.VespaModelUtil; import org.junit.Before; import org.junit.Test; +import java.util.Optional; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.Mockito.doReturn; @@ -71,30 +74,31 @@ public class HostedVespaClusterPolicyTest { @Test public void verifyGroupGoingDownIsFine_noServicesOutsideGroupIsDownIsFine() { - verifyGroupGoingDownIsFine(true, false, 13, true); + verifyGroupGoingDownIsFine(true, Optional.empty(), 13, true); } @Test public void verifyGroupGoingDownIsFine_noServicesInGroupIsUp() { - verifyGroupGoingDownIsFine(false, true, 13, true); + var reasons = new SuspensionReasons().addReason(new HostName("host1"), "supension reason 1"); + verifyGroupGoingDownIsFine(false, Optional.of(reasons), 13, true); } @Test public void verifyGroupGoingDownIsFine_percentageIsFine() { - verifyGroupGoingDownIsFine(false, false, 9, true); + verifyGroupGoingDownIsFine(false, Optional.empty(), 9, true); } @Test public void verifyGroupGoingDownIsFine_fails() { - verifyGroupGoingDownIsFine(false, false, 13, false); + verifyGroupGoingDownIsFine(false, Optional.empty(), 13, false); } private void verifyGroupGoingDownIsFine(boolean noServicesOutsideGroupIsDown, - boolean noServicesInGroupIsUp, + Optional<SuspensionReasons> noServicesInGroupIsUp, int percentageOfServicesDownIfGroupIsAllowedToBeDown, boolean expectSuccess) { when(clusterApi.noServicesOutsideGroupIsDown()).thenReturn(noServicesOutsideGroupIsDown); - when(clusterApi.noServicesInGroupIsUp()).thenReturn(noServicesInGroupIsUp); + when(clusterApi.reasonsForNoServicesInGroupIsUp()).thenReturn(noServicesInGroupIsUp); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(20); doReturn(ConcurrentSuspensionLimitForCluster.TEN_PERCENT).when(policy).getConcurrentSuspensionLimit(clusterApi); @@ -107,12 +111,15 @@ public class HostedVespaClusterPolicyTest { when(clusterApi.getNodeGroup()).thenReturn(nodeGroup); when(nodeGroup.toCommaSeparatedString()).thenReturn("node-group"); - when(clusterApi.noServicesInGroupIsUp()).thenReturn(false); try { - policy.verifyGroupGoingDownIsFine(clusterApi); + SuspensionReasons reasons = policy.verifyGroupGoingDownIsFine(clusterApi); if (!expectSuccess) { fail(); } + + if (noServicesInGroupIsUp.isPresent()) { + assertEquals(noServicesInGroupIsUp.get().getMessagesInOrder(), reasons.getMessagesInOrder()); + } } catch (HostStateChangeDeniedException e) { if (!expectSuccess) { assertEquals("Changing the state of node-group would violate enough-services-up: " + diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index ed6917a3a4e..599b50548b7 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator.policy; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.OrchestratorContext; @@ -34,7 +35,8 @@ public class HostedVespaPolicyTest { private final ClusterControllerClientFactory clientFactory = mock(ClusterControllerClientFactory.class); private final ClusterControllerClient client = mock(ClusterControllerClient.class); - private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3); + private final ManualClock clock = new ManualClock(); + private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, clock); @Before public void setUp() { @@ -44,6 +46,7 @@ public class HostedVespaPolicyTest { @Test public void testGrantSuspension() throws HostStateChangeDeniedException { final HostedVespaClusterPolicy clusterPolicy = mock(HostedVespaClusterPolicy.class); + when(clusterPolicy.verifyGroupGoingDownIsFine(any())).thenReturn(SuspensionReasons.nothingNoteworthy()); final HostedVespaPolicy policy = new HostedVespaPolicy(clusterPolicy, clientFactory, applicationApiFactory); final ApplicationApi applicationApi = mock(ApplicationApi.class); when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("tenant:app:default")); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasonsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasonsTest.java new file mode 100644 index 00000000000..11274dc5d75 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasonsTest.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.orchestrator.policy;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SuspensionReasonsTest { + private final ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600440708123L)); + private final ServiceInstance service = mock(ServiceInstance.class); + private final ServiceInstance service2 = mock(ServiceInstance.class); + + @Test + public void test() { + assertEquals(Optional.empty(), new SuspensionReasons().makeLogMessage()); + assertEquals(Optional.empty(), SuspensionReasons.nothingNoteworthy().makeLogMessage()); + + when(service.hostName()).thenReturn(new HostName("host1")); + when(service.descriptiveName()).thenReturn("service1"); + when(service2.hostName()).thenReturn(new HostName("host2")); + when(service2.descriptiveName()).thenReturn("service2"); + + var reasons = SuspensionReasons.downSince(service, clock.instant(), Duration.ofSeconds(30)); + reasons.mergeWith(SuspensionReasons.isDown(service2)); + assertEquals(Optional.of( + "host1 suspended because service1 has been down since 2020-09-18T14:51:48Z (30 seconds); " + + "host2 suspended because service2 is down"), + reasons.makeLogMessage()); + + } + +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index 0605fcb8a0a..46346e7de7f 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.Policy; +import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest; @@ -83,7 +84,7 @@ public class HostResourceTest { private static final ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZkStatusService( new MockCurator(), mock(Metric.class), new TestTimer(), new DummyAntiServiceMonitor()); - private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3); + private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, clock); static { when(serviceMonitor.getApplication(any(HostName.class))) @@ -107,7 +108,8 @@ public class HostResourceTest { private static class AlwaysAllowPolicy implements Policy { @Override - public void grantSuspensionRequest(OrchestratorContext context, ApplicationApi applicationApi) { + public SuspensionReasons grantSuspensionRequest(OrchestratorContext context, ApplicationApi applicationApi) { + return SuspensionReasons.nothingNoteworthy(); } @Override @@ -208,18 +210,18 @@ public class HostResourceTest { private static class AlwaysFailPolicy implements Policy { @Override - public void grantSuspensionRequest(OrchestratorContext context, ApplicationApi applicationApi) throws HostStateChangeDeniedException { - doThrow(); + public SuspensionReasons grantSuspensionRequest(OrchestratorContext context, ApplicationApi applicationApi) throws HostStateChangeDeniedException { + throw newHostStateChangeDeniedException(); } @Override public void releaseSuspensionGrant(OrchestratorContext context, ApplicationApi application) throws HostStateChangeDeniedException { - doThrow(); + throw newHostStateChangeDeniedException(); } @Override public void acquirePermissionToRemove(OrchestratorContext context, ApplicationApi applicationApi) throws HostStateChangeDeniedException { - doThrow(); + throw newHostStateChangeDeniedException(); } @Override @@ -227,11 +229,11 @@ public class HostResourceTest { OrchestratorContext context, ApplicationInstance applicationInstance, HostName hostName, ApplicationLock hostStatusRegistry) throws HostStateChangeDeniedException { - doThrow(); + throw newHostStateChangeDeniedException(); } - private static void doThrow() throws HostStateChangeDeniedException { - throw new HostStateChangeDeniedException( + private static HostStateChangeDeniedException newHostStateChangeDeniedException() { + return new HostStateChangeDeniedException( new HostName("some-host"), "impossible-policy", "This policy rejects all requests"); diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index fe0504b25af..f29712e7a5f 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -48,6 +48,7 @@ vespa_define_module( src/apps/vespa-dump-feed src/apps/vespa-gen-testdocs src/apps/vespa-proton-cmd + src/apps/vespa-spi-feed-bm src/apps/vespa-transactionlog-inspect TESTS diff --git a/searchcore/src/apps/vespa-spi-feed-bm/.gitignore b/searchcore/src/apps/vespa-spi-feed-bm/.gitignore new file mode 100644 index 00000000000..02fff2fe280 --- /dev/null +++ b/searchcore/src/apps/vespa-spi-feed-bm/.gitignore @@ -0,0 +1 @@ +vespa-spi-feed-bm diff --git a/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt b/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt new file mode 100644 index 00000000000..e188bc16ec0 --- /dev/null +++ b/searchcore/src/apps/vespa-spi-feed-bm/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_vespa_spi_feed_bm_app + SOURCES + vespa_spi_feed_bm.cpp + OUTPUT_NAME vespa-spi-feed-bm + DEPENDS + searchcore_server + searchcore_initializer + searchcore_reprocessing + searchcore_index + searchcore_persistenceengine + searchcore_docsummary + searchcore_feedoperation + searchcore_matching + searchcore_attribute + searchcore_documentmetastore + searchcore_bucketdb + searchcore_flushengine + searchcore_pcommon + searchcore_grouping + searchcore_proton_metrics + searchcore_fconfig + searchlib_searchlib_uca +) diff --git a/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp b/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp new file mode 100644 index 00000000000..09a028d0431 --- /dev/null +++ b/searchcore/src/apps/vespa-spi-feed-bm/vespa_spi_feed_bm.cpp @@ -0,0 +1,653 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> + +#include <tests/proton/common/dummydbowner.h> +#include <vespa/config-imported-fields.h> +#include <vespa/config-rank-profiles.h> +#include <vespa/config-summarymap.h> +#include <vespa/fastos/file.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/searchcommon/common/schemaconfigurer.h> +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/matching/querylimiter.h> +#include <vespa/searchcore/proton/metrics/metricswireservice.h> +#include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h> +#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> +#include <vespa/searchcore/proton/server/bootstrapconfig.h> +#include <vespa/searchcore/proton/server/document_db_maintenance_config.h> +#include <vespa/searchcore/proton/server/documentdb.h> +#include <vespa/searchcore/proton/server/documentdbconfigmanager.h> +#include <vespa/searchcore/proton/server/fileconfigmanager.h> +#include <vespa/searchcore/proton/server/memoryconfigstore.h> +#include <vespa/searchcore/proton/server/persistencehandlerproxy.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/transactionlog/translogserver.h> +#include <vespa/searchsummary/config/config-juniperrc.h> +#include <vespa/vespalib/util/lambdatask.h> +#include <vespa/config-bucketspaces.h> +#include <vespa/config-attributes.h> +#include <vespa/config-indexschema.h> +#include <vespa/config-summary.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/fastos/app.h> +#include <getopt.h> +#include <iostream> + +#include <vespa/log/log.h> +LOG_SETUP("vespa-spi-feed-bm"); + +using namespace config; +using namespace proton; +using namespace cloud::config::filedistribution; +using namespace vespa::config::search::core; +using namespace vespa::config::search::summary; +using namespace vespa::config::search; +using namespace std::chrono_literals; +using vespa::config::content::core::BucketspacesConfig; + +using document::AssignValueUpdate; +using document::BucketId; +using document::BucketSpace; +using document::Document; +using document::DocumentId; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::DocumenttypesConfig; +using document::DocumentUpdate; +using document::Field; +using document::FieldUpdate; +using document::IntFieldValue; +using document::test::makeBucketSpace; +using search::TuneFileDocumentDB; +using search::index::DummyFileHeaderContext; +using search::index::Schema; +using search::index::SchemaBuilder; +using search::transactionlog::TransLogServer; +using storage::spi::Bucket; +using storage::spi::PartitionId; +using storage::spi::PersistenceProvider; +using storage::spi::Priority; +using storage::spi::Timestamp; +using storage::spi::Trace; +using vespalib::makeLambdaTask; + +using DocumentDBMap = std::map<DocTypeName, std::shared_ptr<DocumentDB>>; + +namespace { + +storage::spi::LoadType default_load_type(0, "default"); + +vespalib::string base_dir = "testdb"; + +std::shared_ptr<DocumenttypesConfig> make_document_type() { + using Struct = document::config_builder::Struct; + using DataType = document::DataType; + document::config_builder::DocumenttypesConfigBuilderHelper builder; + builder.document(42, "test", Struct("test.header").addField("int", DataType::T_INT), Struct("test.body")); + return std::make_shared<DocumenttypesConfig>(builder.config()); +} + +std::shared_ptr<AttributesConfig> make_attributes_config() { + AttributesConfigBuilder builder; + AttributesConfig::Attribute attribute; + attribute.name = "int"; + attribute.datatype = AttributesConfig::Attribute::Datatype::INT32; + builder.attribute.emplace_back(attribute); + return std::make_shared<AttributesConfig>(builder); +} + +std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<DocumenttypesConfig> document_types, std::shared_ptr<const DocumentTypeRepo> repo, const DocTypeName& doc_type_name) +{ + auto indexschema = std::make_shared<IndexschemaConfig>(); + auto attributes = make_attributes_config(); + auto summary = std::make_shared<SummaryConfig>(); + std::shared_ptr<Schema> schema(new Schema()); + SchemaBuilder::build(*indexschema, *schema); + SchemaBuilder::build(*attributes, *schema); + SchemaBuilder::build(*summary, *schema); + return std::make_shared<DocumentDBConfig>( + 1, + std::make_shared<RankProfilesConfig>(), + std::make_shared<matching::RankingConstants>(), + std::make_shared<matching::OnnxModels>(), + indexschema, + attributes, + summary, + std::make_shared<SummarymapConfig>(), + std::make_shared<JuniperrcConfig>(), + document_types, + repo, + std::make_shared<ImportedFieldsConfig>(), + std::make_shared<TuneFileDocumentDB>(), + schema, + std::make_shared<DocumentDBMaintenanceConfig>(), + search::LogDocumentStore::Config(), + "client", + doc_type_name.getName()); +} + +class MyPersistenceEngineOwner : public IPersistenceEngineOwner +{ + void setClusterState(BucketSpace, const storage::spi::ClusterState &) override { } +}; + +struct MyResourceWriteFilter : public IResourceWriteFilter +{ + bool acceptWriteOperation() const override { return true; } + State getAcceptState() const override { return IResourceWriteFilter::State(); } +}; + +class MyPendingTracker { + uint32_t _pending; + uint32_t _limit; + std::mutex _mutex; + std::condition_variable _cond; + +public: + MyPendingTracker(uint32_t limit) + : _pending(0u), + _limit(limit), + _mutex(), + _cond() + { + } + + ~MyPendingTracker() + { + drain(); + } + + void release() { + std::unique_lock<std::mutex> guard(_mutex); + --_pending; + if (_pending < _limit) { + _cond.notify_all(); + } + //LOG(info, "release, pending is now %u", _pending); + } + void retain() { + std::unique_lock<std::mutex> guard(_mutex); + while (_pending >= _limit) { + _cond.wait(guard); + } + ++_pending; + //LOG(info, "retain, pending is now %u", _pending); + } + + void drain() { + std::unique_lock<std::mutex> guard(_mutex); + while (_pending > 0) { + _cond.wait(guard); + } + } +}; + +class MyOperationComplete : public storage::spi::OperationComplete +{ + MyPendingTracker& _tracker; +public: + MyOperationComplete(MyPendingTracker &tracker); + ~MyOperationComplete(); + void onComplete(std::unique_ptr<storage::spi::Result> result) override; + void addResultHandler(const storage::spi::ResultHandler* resultHandler) override; +}; + +MyOperationComplete::MyOperationComplete(MyPendingTracker& tracker) + : _tracker(tracker) +{ + _tracker.retain(); +} + +MyOperationComplete::~MyOperationComplete() +{ + _tracker.release(); +} + +void +MyOperationComplete::onComplete(std::unique_ptr<storage::spi::Result> result) +{ + (void) result; +} + +void +MyOperationComplete::addResultHandler(const storage::spi::ResultHandler * resultHandler) +{ + (void) resultHandler; +} + +class BMRange +{ + uint32_t _start; + uint32_t _end; +public: + BMRange(uint32_t start_in, uint32_t end_in) + : _start(start_in), + _end(end_in) + { + } + uint32_t get_start() const { return _start; } + uint32_t get_end() const { return _end; } +}; + +class BMParams { + uint32_t _documents; + uint32_t _threads; + uint32_t _update_passes; + uint32_t get_start(uint32_t thread_id) const { + return (_documents / _threads) * thread_id + std::min(thread_id, _documents % _threads); + } +public: + BMParams() + : _documents(160000), + _threads(32), + _update_passes(1) + { + } + BMRange get_range(uint32_t thread_id) const { + return BMRange(get_start(thread_id), get_start(thread_id + 1)); + } + uint32_t get_documents() const { return _documents; } + uint32_t get_threads() const { return _threads; } + uint32_t get_update_passes() const { return _update_passes; } + void set_documents(uint32_t documents_in) { _documents = documents_in; } + void set_threads(uint32_t threads_in) { _threads = threads_in; } + void set_update_passes(uint32_t update_passes_in) { _update_passes = update_passes_in; } + bool check() const; +}; + +bool +BMParams::check() const +{ + if (_threads < 1) { + std::cerr << "Too few threads: " << _threads << std::endl; + return false; + } + if (_threads > 256) { + std::cerr << "Too many threads: " << _threads << std::endl; + return false; + } + if (_documents < _threads) { + std::cerr << "Too few documents: " << _documents << std::endl; + return false; + } + return true; +} + +} + + +struct PersistenceProviderFixture { + std::shared_ptr<DocumenttypesConfig> _document_types; + std::shared_ptr<const DocumentTypeRepo> _repo; + DocTypeName _doc_type_name; + const DocumentType* _document_type; + const Field& _field; + std::shared_ptr<DocumentDBConfig> _document_db_config; + vespalib::string _base_dir; + DummyFileHeaderContext _file_header_context; + int _tls_listen_port; + TransLogServer _tls; + vespalib::string _tls_spec; + matching::QueryLimiter _query_limiter; + vespalib::Clock _clock; + DummyWireService _metrics_wire_service; + MemoryConfigStores _config_stores; + vespalib::ThreadStackExecutor _summary_executor; + DummyDBOwner _document_db_owner; + BucketSpace _bucket_space; + std::shared_ptr<DocumentDB> _document_db; + MyPersistenceEngineOwner _persistence_owner; + MyResourceWriteFilter _write_filter; + std::shared_ptr<PersistenceEngine> _persistence_engine; + storage::spi::Context _context; + uint32_t _bucket_bits; + + PersistenceProviderFixture(); + ~PersistenceProviderFixture(); + void create_document_db(); + uint32_t num_buckets() const { return (1u << _bucket_bits); } + Bucket make_bucket(uint32_t i) const { return Bucket(document::Bucket(_bucket_space, BucketId(_bucket_bits, i & (num_buckets() - 1))), PartitionId(0)); } + DocumentId make_document_id(uint32_t i) const; + std::unique_ptr<Document> make_document(uint32_t i) const; + std::unique_ptr<DocumentUpdate> make_document_update(uint32_t i) const; + void create_buckets(); +}; + +PersistenceProviderFixture::PersistenceProviderFixture() + : _document_types(make_document_type()), + _repo(std::make_shared<DocumentTypeRepo>(*_document_types)), + _doc_type_name("test"), + _document_type(_repo->getDocumentType(_doc_type_name.getName())), + _field(_document_type->getField("int")), + _document_db_config(make_document_db_config(_document_types, _repo, _doc_type_name)), + _base_dir(base_dir), + _file_header_context(), + _tls_listen_port(9017), + _tls("tls", _tls_listen_port, _base_dir, _file_header_context), + _tls_spec(vespalib::make_string("tcp/localhost:%d", _tls_listen_port)), + _query_limiter(), + _clock(), + _metrics_wire_service(), + _config_stores(), + _summary_executor(8, 128 * 1024), + _document_db_owner(), + _bucket_space(makeBucketSpace(_doc_type_name.getName())), + _document_db(), + _persistence_owner(), + _write_filter(), + _persistence_engine(), + _context(default_load_type, Priority(0), Trace::TraceLevel(0)), + _bucket_bits(16) +{ + create_document_db(); + _persistence_engine = std::make_unique<PersistenceEngine>(_persistence_owner, _write_filter, -1, false); + auto proxy = std::make_shared<PersistenceHandlerProxy>(_document_db); + _persistence_engine->putHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name, proxy); +} + +PersistenceProviderFixture::~PersistenceProviderFixture() +{ + if (_persistence_engine) { + _persistence_engine->destroyIterators(); + _persistence_engine->removeHandler(_persistence_engine->getWLock(), _bucket_space, _doc_type_name); + } + if (_document_db) { + _document_db->close(); + } +} + +void +PersistenceProviderFixture::create_document_db() +{ + vespalib::mkdir(_base_dir, false); + vespalib::mkdir(_base_dir + "/" + _doc_type_name.getName(), false); + vespalib::string input_cfg = _base_dir + "/" + _doc_type_name.getName() + "/baseconfig"; + { + FileConfigManager fileCfg(input_cfg, "", _doc_type_name.getName()); + fileCfg.saveConfig(*_document_db_config, 1); + } + config::DirSpec spec(input_cfg + "/config-1"); + auto tuneFileDocDB = std::make_shared<TuneFileDocumentDB>(); + DocumentDBConfigHelper mgr(spec, _doc_type_name.getName()); + auto bootstrap_config = std::make_shared<BootstrapConfig>(1, + _document_types, + _repo, + std::make_shared<ProtonConfig>(), + std::make_shared<FiledistributorrpcConfig>(), + std::make_shared<BucketspacesConfig>(), + tuneFileDocDB, HwInfo()); + mgr.forwardConfig(bootstrap_config); + mgr.nextGeneration(0ms); + _document_db = std::make_shared<DocumentDB>(_base_dir, + mgr.getConfig(), + _tls_spec, + _query_limiter, + _clock, + _doc_type_name, + _bucket_space, + *bootstrap_config->getProtonConfigSP(), + _document_db_owner, + _summary_executor, + _summary_executor, + _tls, + _metrics_wire_service, + _file_header_context, + _config_stores.getConfigStore(_doc_type_name.toString()), + std::make_shared<vespalib::ThreadStackExecutor>(16, 128 * 1024), + HwInfo()); + _document_db->start(); + _document_db->waitForOnlineState(); +} + +DocumentId +PersistenceProviderFixture::make_document_id(uint32_t i) const +{ + DocumentId id(vespalib::make_string("id::test:n=%u:%u", i & (num_buckets() - 1), i)); + return id; +} + +std::unique_ptr<Document> +PersistenceProviderFixture::make_document(uint32_t i) const +{ + auto id = make_document_id(i); + auto document = std::make_unique<Document>(*_document_type, id); + document->setRepo(*_repo); + document->setFieldValue(_field, std::make_unique<IntFieldValue>(i)); + return document; +} + +std::unique_ptr<DocumentUpdate> +PersistenceProviderFixture::make_document_update(uint32_t i) const +{ + auto id = make_document_id(i); + auto document_update = std::make_unique<DocumentUpdate>(*_repo, *_document_type, id); + document_update->addUpdate(FieldUpdate(_field).addUpdate(AssignValueUpdate(IntFieldValue(15)))); + return document_update; +} + +void +PersistenceProviderFixture::create_buckets() +{ + auto &provider = *_persistence_engine; + for (unsigned int i = 0; i < num_buckets(); ++i) { + provider.createBucket(make_bucket(i), _context); + } +} + +void +put_async_task(PersistenceProviderFixture &f, BMRange range, int64_t time_bias) +{ + LOG(debug, "put_async_task([%u..%u))", range.get_start(), range.get_end()); + MyPendingTracker pending_tracker(100); + auto &provider = *f._persistence_engine; + auto &context = f._context; + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + auto bucket = f.make_bucket(i); + auto document = f.make_document(i); + provider.putAsync(bucket, Timestamp(time_bias + i), std::move(document), context, std::make_unique<MyOperationComplete>(pending_tracker)); + } + pending_tracker.drain(); +} + +void +run_put_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const BMParams& bm_params) +{ + LOG(info, "putAsync %u small documents, pass=%u", bm_params.get_documents(), pass); + auto start_time = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { + auto range = bm_params.get_range(i); + executor.execute(makeLambdaTask([&f, range, time_bias]() + { put_async_task(f, range, time_bias); })); + } + executor.sync(); + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration<double> elapsed = end_time - start_time; + LOG(info, "%8.2f puts/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); + time_bias += bm_params.get_documents(); +} + +void +update_async_task(PersistenceProviderFixture &f, BMRange range, int64_t time_bias) +{ + LOG(debug, "update_async_task([%u..%u))", range.get_start(), range.get_end()); + MyPendingTracker pending_tracker(100); + auto &provider = *f._persistence_engine; + auto &context = f._context; + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + auto bucket = f.make_bucket(i); + auto document_update = f.make_document_update(i); + provider.updateAsync(bucket, Timestamp(time_bias + i), std::move(document_update), context, std::make_unique<MyOperationComplete>(pending_tracker)); + } + pending_tracker.drain(); +} + +void +run_update_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const BMParams& bm_params) +{ + LOG(info, "updateAsync %u small documents, pass=%u", bm_params.get_documents(), pass); + auto start_time = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { + auto range = bm_params.get_range(i); + executor.execute(makeLambdaTask([&f, range, time_bias]() + { update_async_task(f, range, time_bias); })); + } + executor.sync(); + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration<double> elapsed = end_time - start_time; + LOG(info, "%8.2f updates/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); + time_bias += bm_params.get_documents(); +} + +void +remove_async_task(PersistenceProviderFixture &f, BMRange range, int64_t time_bias) +{ + LOG(debug, "remove_async_task([%u..%u))", range.get_start(), range.get_end()); + MyPendingTracker pending_tracker(100); + auto &provider = *f._persistence_engine; + auto &context = f._context; + for (unsigned int i = range.get_start(); i < range.get_end(); ++i) { + auto bucket = f.make_bucket(i); + auto document_id = f.make_document_id(i); + provider.removeAsync(bucket, Timestamp(time_bias + i), document_id, context, std::make_unique<MyOperationComplete>(pending_tracker)); + } + pending_tracker.drain(); +} + +void +run_remove_async_tasks(PersistenceProviderFixture &f, vespalib::ThreadStackExecutor &executor, int pass, int64_t& time_bias, const BMParams &bm_params) +{ + LOG(info, "removeAsync %u small documents, pass=%u", bm_params.get_documents(), pass); + auto start_time = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < bm_params.get_threads(); ++i) { + auto range = bm_params.get_range(i); + executor.execute(makeLambdaTask([&f, range, time_bias]() + { remove_async_task(f, range, time_bias); })); + } + executor.sync(); + auto end_time = std::chrono::steady_clock::now(); + std::chrono::duration<double> elapsed = end_time - start_time; + LOG(info, "%8.2f removes/s for pass=%u", bm_params.get_documents() / elapsed.count(), pass); + time_bias += bm_params.get_documents(); +} + +void benchmark_async_spi(const BMParams &bm_params) +{ + vespalib::rmdir(base_dir, true); + PersistenceProviderFixture f; + auto &provider = *f._persistence_engine; + LOG(info, "start initialize"); + provider.initialize(); + LOG(info, "create %u buckets", f.num_buckets()); + f.create_buckets(); + vespalib::ThreadStackExecutor executor(bm_params.get_threads(), 128 * 1024); + int64_t time_bias = 1; + run_put_async_tasks(f, executor, 0, time_bias, bm_params); + run_put_async_tasks(f, executor, 1, time_bias, bm_params); + for (uint32_t pass = 0; pass < bm_params.get_update_passes(); ++pass) { + run_update_async_tasks(f, executor, pass, time_bias, bm_params); + } + run_remove_async_tasks(f, executor, 0, time_bias, bm_params); + run_remove_async_tasks(f, executor, 1, time_bias, bm_params); +} + +class App : public FastOS_Application +{ + BMParams _bm_params; +public: + App(); + ~App() override; + void usage(); + bool get_options(); + int Main() override; +}; + +App::App() + : _bm_params() +{ +} + +App::~App() = default; + +void +App::usage() +{ + std::cerr << + "vespa-spi-feed-bm version 0.0\n" + "\n" + "USAGE:\n"; + std::cerr << + "vespa-spi-feed-bm\n" + "[--threads threads]\n" + "[--documents documents]" + "[--update-passes update-passes]" << std::endl; +} + +bool +App::get_options() +{ + int c; + const char *opt_argument = nullptr; + int long_opt_index = 0; + static struct option long_opts[] = { + { "threads", 1, nullptr, 0 }, + { "documents", 1, nullptr, 0 }, + { "update-passes", 1, nullptr, 0 } + }; + enum longopts_enum { + LONGOPT_THREADS, + LONGOPT_DOCUMENTS, + LONGOPT_UPDATE_PASSES + }; + int opt_index = 1; + resetOptIndex(opt_index); + while ((c = GetOptLong("", opt_argument, opt_index, long_opts, &long_opt_index)) != -1) { + switch (c) { + case 0: + switch(long_opt_index) { + case LONGOPT_THREADS: + _bm_params.set_threads(atoi(opt_argument)); + break; + case LONGOPT_DOCUMENTS: + _bm_params.set_documents(atoi(opt_argument)); + break; + case LONGOPT_UPDATE_PASSES: + _bm_params.set_update_passes(atoi(opt_argument)); + break; + default: + return false; + } + break; + default: + return false; + } + } + return _bm_params.check(); +} + +int +App::Main() +{ + if (!get_options()) { + usage(); + return 1; + } + benchmark_async_spi(_bm_params); + return 0; +} + +int +main(int argc, char* argv[]) +{ + DummyFileHeaderContext::setCreator("vespa-spi-feed-bm"); + App app; + auto exit_value = app.Entry(argc, argv); + vespalib::rmdir(base_dir, true); + return exit_value; +} diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 2f34292ad52..a7c53293981 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -286,7 +286,7 @@ public: * and dispatches each packet entry to the ReplayPacketDispatcher that * transforms them into concrete operations. */ -class VisitorCallback : public TransLogClient::Session::Callback +class VisitorCallback : public client::Callback { private: ReplayPacketDispatcher _dispatcher; @@ -298,7 +298,7 @@ public: _eof(false) { } - virtual RPC::Result receive(const Packet &packet) override { + client::RPC::Result receive(const Packet &packet) override { vespalib::nbostream_longlivedbuf handle(packet.getHandle().data(), packet.getHandle().size()); try { while (handle.size() > 0) { @@ -309,11 +309,11 @@ public: } catch (const std::exception &e) { std::cerr << "Error while handling transaction log packet: '" << std::string(e.what()) << "'" << std::endl; - return RPC::ERROR; + return client::RPC::ERROR; } - return RPC::OK; + return client::RPC::OK; } - virtual void eof() override { _eof = true; } + void eof() override { _eof = true; } bool isEof() const { return _eof; } }; @@ -371,7 +371,7 @@ protected: const BaseOptions &_bopts; DummyFileHeaderContext _fileHeader; TransLogServer _server; - TransLogClient _client; + client::TransLogClient _client; public: BaseUtility(const BaseOptions &bopts) @@ -416,7 +416,7 @@ public: _client.listDomains(domains); std::cout << "Listing status for " << domains.size() << " domain(s):" << std::endl; for (size_t i = 0; i < domains.size(); ++i) { - TransLogClient::Session::UP session = _client.open(domains[i]); + std::unique_ptr<client::Session> session = _client.open(domains[i]); SerialNum first; SerialNum last; size_t count; @@ -484,7 +484,7 @@ protected: DocTypeRepo repo(_oopts.configDir); IReplayPacketHandlerUP handler = createHandler(repo.docTypeRepo); VisitorCallback callback(*handler); - TransLogClient::Visitor::UP visitor = _client.createVisitor(_oopts.domainName, callback); + std::unique_ptr<client::Visitor> visitor = _client.createVisitor(_oopts.domainName, callback); bool visitOk = visitor->visit(_oopts.firstSerialNum-1, _oopts.lastSerialNum); if (!visitOk) { std::cerr << "Visiting domain '" << _oopts.domainName << "' [" << _oopts.firstSerialNum << "," diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 92117e174e9..8dea4ba67ba 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -17,6 +17,7 @@ #include <vespa/searchcore/proton/metrics/metricswireservice.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/documentdb.h> +#include <vespa/searchcore/proton/server/feedhandler.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/server/idocumentsubdb.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> @@ -256,11 +257,14 @@ public: storage::spi::Timestamp ts(0); DbDocumentId dbdId(lid); DbDocumentId prevDbdId(0); - PutOperation op(bucketId, ts, std::make_shared<document::Document>(doc)); - op.setSerialNum(serialNum); - op.setDbDocumentId(dbdId); - op.setPrevDbDocumentId(prevDbdId); - _ddb->getFeedHandler().storeOperation(op, std::make_shared<search::IgnoreCallback>()); + auto op = std::make_unique<PutOperation>(bucketId, ts, std::make_shared<document::Document>(doc)); + op->setSerialNum(serialNum); + op->setDbDocumentId(dbdId); + op->setPrevDbDocumentId(prevDbdId); + _ddb->getWriteService().master().execute(vespalib::makeLambdaTask([this, op = std::move(op)]() { + _ddb->getFeedHandler().storeOperation(*op, std::make_shared<search::IgnoreCallback>()); + })); + _ddb->getWriteService().master().sync(); SearchView *sv(dynamic_cast<SearchView *>(_ddb->getReadySubDB()->getSearchView().get())); if (sv != nullptr) { // cf. FeedView::putAttributes() @@ -271,13 +275,17 @@ public: } }; -class Test : public vespalib::TestApp +class Fixture { private: std::unique_ptr<vespa::config::search::SummaryConfig> _summaryCfg; - ResultConfig _resultCfg; + ResultConfig _resultCfg; std::set<vespalib::string> _markupFields; +public: + Fixture(); + ~Fixture(); + const ResultConfig &getResultConfig() const{ return _resultCfg; } @@ -285,43 +293,10 @@ private: const std::set<vespalib::string> &getMarkupFields() const{ return _markupFields; } - - GeneralResultPtr - getResult(DocumentStoreAdapter & dsa, uint32_t docId); - - GeneralResultPtr - getResult(const DocsumReply & reply, uint32_t id, uint32_t resultClassID); - - bool assertString(const std::string & exp, const std::string & fieldName, DocumentStoreAdapter &dsa, uint32_t id); - - void assertTensor(const Tensor::UP &exp, const std::string &fieldName, const DocsumReply &reply, - uint32_t id, uint32_t resultClassID); - - bool assertSlime(const std::string &exp, const DocsumReply &reply, uint32_t id,bool relaxed = false); - - void requireThatAdapterHandlesAllFieldTypes(); - void requireThatAdapterHandlesMultipleDocuments(); - void requireThatAdapterHandlesDocumentIdField(); - void requireThatDocsumRequestIsProcessed(); - void requireThatRewritersAreUsed(); - void requireThatAttributesAreUsed(); - void requireThatSummaryAdapterHandlesPutAndRemove(); - void requireThatAnnotationsAreUsed(); - void requireThatUrisAreUsed(); - void requireThatPositionsAreUsed(); - void requireThatRawFieldsWorks(); - void requireThatFieldCacheRepoCanReturnDefaultFieldCache(); - void requireThatSummariesTimeout(); - -public: - Test(); - ~Test(); - int Main() override; }; - GeneralResultPtr -Test::getResult(DocumentStoreAdapter & dsa, uint32_t docId) +getResult(DocumentStoreAdapter & dsa, uint32_t docId) { DocsumStoreValue docsum = dsa.getMappedDocsum(docId); ASSERT_TRUE(docsum.pt() != nullptr); @@ -331,20 +306,8 @@ Test::getResult(DocumentStoreAdapter & dsa, uint32_t docId) return retval; } - -GeneralResultPtr -Test::getResult(const DocsumReply & reply, uint32_t id, uint32_t resultClassID) -{ - auto retval = std::make_unique<GeneralResult>(getResultConfig().LookupResultClass(resultClassID)); - const DocsumReply::Docsum & docsum = reply.docsums[id]; - // skip the 4 byte class id - ASSERT_TRUE(retval->unpack(docsum.data.c_str() + 4, docsum.data.size() - 4)); - return retval; -} - - bool -Test::assertString(const std::string & exp, const std::string & fieldName, +assertString(const std::string & exp, const std::string & fieldName, DocumentStoreAdapter &dsa, uint32_t id) { GeneralResultPtr res = getResult(dsa, id); @@ -353,7 +316,7 @@ Test::assertString(const std::string & exp, const std::string & fieldName, } void -Test::assertTensor(const Tensor::UP & exp, const std::string & fieldName, +assertTensor(const Tensor::UP & exp, const std::string & fieldName, const DocsumReply & reply, uint32_t id, uint32_t) { const DocsumReply::Docsum & docsum = reply.docsums[id]; @@ -403,7 +366,7 @@ getSlime(const DocsumReply &reply, uint32_t id, bool relaxed) } bool -Test::assertSlime(const std::string &exp, const DocsumReply &reply, uint32_t id, bool relaxed) +assertSlime(const std::string &exp, const DocsumReply &reply, uint32_t id, bool relaxed) { vespalib::Slime slime = getSlime(reply, id, relaxed); vespalib::Slime expSlime; @@ -412,8 +375,7 @@ Test::assertSlime(const std::string &exp, const DocsumReply &reply, uint32_t id, return EXPECT_EQUAL(expSlime, slime); } -void -Test::requireThatAdapterHandlesAllFieldTypes() +TEST_F("requireThatAdapterHandlesAllFieldTypes", Fixture) { Schema s; s.addSummaryField(Schema::SummaryField("a", schema::DataType::INT8)); @@ -447,9 +409,9 @@ Test::requireThatAdapterHandlesAllFieldTypes() DocumentStoreAdapter dsa(bc._str, *bc._repo, - getResultConfig(), "class0", - bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"), - getMarkupFields()); + f.getResultConfig(), "class0", + bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class0"), + f.getMarkupFields()); GeneralResultPtr res = getResult(dsa, 0); EXPECT_EQUAL(255u, res->GetEntry("a")->_intval); EXPECT_EQUAL(32767u, res->GetEntry("b")->_intval); @@ -472,8 +434,7 @@ Test::requireThatAdapterHandlesAllFieldTypes() } -void -Test::requireThatAdapterHandlesMultipleDocuments() +TEST_F("requireThatAdapterHandlesMultipleDocuments", Fixture) { Schema s; s.addSummaryField(Schema::SummaryField("a", schema::DataType::INT32)); @@ -489,9 +450,9 @@ Test::requireThatAdapterHandlesMultipleDocuments() addInt(2000).endField(); bc.endDocument(1); - DocumentStoreAdapter dsa(bc._str, *bc._repo, getResultConfig(), "class1", - bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class1"), - getMarkupFields()); + DocumentStoreAdapter dsa(bc._str, *bc._repo, f.getResultConfig(), "class1", + bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class1"), + f.getMarkupFields()); { // doc 0 GeneralResultPtr res = getResult(dsa, 0); EXPECT_EQUAL(1000u, res->GetEntry("a")->_intval); @@ -514,8 +475,7 @@ Test::requireThatAdapterHandlesMultipleDocuments() } -void -Test::requireThatAdapterHandlesDocumentIdField() +TEST_F("requireThatAdapterHandlesDocumentIdField", Fixture) { Schema s; s.addSummaryField(Schema::SummaryField("documentid", schema::DataType::STRING)); @@ -525,9 +485,9 @@ Test::requireThatAdapterHandlesDocumentIdField() addStr("foo"). endField(); bc.endDocument(0); - DocumentStoreAdapter dsa(bc._str, *bc._repo, getResultConfig(), "class4", - bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class4"), - getMarkupFields()); + DocumentStoreAdapter dsa(bc._str, *bc._repo, f.getResultConfig(), "class4", + bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class4"), + f.getMarkupFields()); GeneralResultPtr res = getResult(dsa, 0); EXPECT_EQUAL("id:ns:searchdocument::0", std::string(res->GetEntry("documentid")->_stringval, res->GetEntry("documentid")->_stringlen)); @@ -540,8 +500,7 @@ GlobalId gid3 = DocumentId("id:ns:searchdocument::3").getGlobalId(); // lid 3 GlobalId gid4 = DocumentId("id:ns:searchdocument::4").getGlobalId(); // lid 4 GlobalId gid9 = DocumentId("id:ns:searchdocument::9").getGlobalId(); // not existing -void -Test::requireThatDocsumRequestIsProcessed() +TEST("requireThatDocsumRequestIsProcessed") { Schema s; s.addSummaryField(Schema::SummaryField("a", schema::DataType::INT32)); @@ -599,8 +558,7 @@ Test::requireThatDocsumRequestIsProcessed() } -void -Test::requireThatRewritersAreUsed() +TEST("requireThatRewritersAreUsed") { Schema s; s.addSummaryField(Schema::SummaryField("aa", schema::DataType::INT32)); @@ -626,8 +584,7 @@ Test::requireThatRewritersAreUsed() EXPECT_TRUE(assertSlime("{aa:20}", *rep, 0, false)); } -void -Test::requireThatSummariesTimeout() +TEST("requireThatSummariesTimeout") { Schema s; s.addSummaryField(Schema::SummaryField("aa", schema::DataType::INT32)); @@ -671,8 +628,7 @@ addField(Schema & s, } -void -Test::requireThatAttributesAreUsed() +TEST("requireThatAttributesAreUsed") { Schema s; addField(s, "ba", schema::DataType::INT32, CollectionType::SINGLE); @@ -815,8 +771,7 @@ Test::requireThatAttributesAreUsed() } -void -Test::requireThatSummaryAdapterHandlesPutAndRemove() +TEST("requireThatSummaryAdapterHandlesPutAndRemove") { Schema s; s.addSummaryField(Schema::SummaryField("f1", schema::DataType::STRING, CollectionType::SINGLE)); @@ -848,8 +803,7 @@ namespace const std::string empty; } -void -Test::requireThatAnnotationsAreUsed() +TEST_F("requireThatAnnotationsAreUsed", Fixture) { Schema s; s.addIndexField(Schema::IndexField("g", schema::DataType::STRING, CollectionType::SINGLE)); @@ -889,9 +843,9 @@ Test::requireThatAnnotationsAreUsed() EXPECT_EQUAL("foo bar", act->getValue("g")->getAsString()); EXPECT_EQUAL("foo bar", act->getValue("dynamicstring")->getAsString()); - DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0", - bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"), - getMarkupFields()); + DocumentStoreAdapter dsa(store, *bc._repo, f.getResultConfig(), "class0", + bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class0"), + f.getMarkupFields()); EXPECT_TRUE(assertString("foo bar", "g", dsa, 1)); EXPECT_TRUE(assertString(TERM_EMPTY + "foo" + TERM_SEP + " " + TERM_SEP + @@ -900,8 +854,7 @@ Test::requireThatAnnotationsAreUsed() "dynamicstring", dsa, 1)); } -void -Test::requireThatUrisAreUsed() +TEST_F("requireThatUrisAreUsed", Fixture) { Schema s; s.addUriIndexFields(Schema::IndexField("urisingle", schema::DataType::STRING, CollectionType::SINGLE)); @@ -1041,9 +994,9 @@ Test::requireThatUrisAreUsed() EXPECT_TRUE(act.get() != nullptr); EXPECT_EQUAL(exp->getType(), act->getType()); - DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0", - bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"), - getMarkupFields()); + DocumentStoreAdapter dsa(store, *bc._repo, f.getResultConfig(), "class0", + bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class0"), + f.getMarkupFields()); EXPECT_TRUE(assertString("http://www.example.com:81/fluke?ab=2#4", "urisingle", dsa, 1)); GeneralResultPtr res = getResult(dsa, 1); @@ -1068,8 +1021,7 @@ Test::requireThatUrisAreUsed() } -void -Test::requireThatPositionsAreUsed() +TEST("requireThatPositionsAreUsed") { Schema s; s.addAttributeField(Schema::AttributeField("sp2", schema::DataType::INT64)); @@ -1120,8 +1072,7 @@ Test::requireThatPositionsAreUsed() } -void -Test::requireThatRawFieldsWorks() +TEST_F("requireThatRawFieldsWorks", Fixture) { Schema s; s.addSummaryField(Schema::AttributeField("i", schema::DataType::RAW)); @@ -1178,9 +1129,9 @@ Test::requireThatRawFieldsWorks() EXPECT_TRUE(act.get() != nullptr); EXPECT_EQUAL(exp->getType(), act->getType()); - DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0", - bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"), - getMarkupFields()); + DocumentStoreAdapter dsa(store, *bc._repo, f.getResultConfig(), "class0", + bc.createFieldCacheRepo(f.getResultConfig())->getFieldCache("class0"), + f.getMarkupFields()); ASSERT_TRUE(assertString(raw1s, "i", dsa, 1)); @@ -1206,13 +1157,12 @@ Test::requireThatRawFieldsWorks() } -void -Test::requireThatFieldCacheRepoCanReturnDefaultFieldCache() +TEST_F("requireThatFieldCacheRepoCanReturnDefaultFieldCache", Fixture) { Schema s; s.addSummaryField(Schema::SummaryField("a", schema::DataType::INT32)); BuildContext bc(s); - FieldCacheRepo::UP repo = bc.createFieldCacheRepo(getResultConfig()); + FieldCacheRepo::UP repo = bc.createFieldCacheRepo(f.getResultConfig()); FieldCache::CSP cache = repo->getFieldCache(""); EXPECT_TRUE(cache.get() == repo->getFieldCache("class1").get()); EXPECT_EQUAL(1u, cache->size()); @@ -1220,7 +1170,7 @@ Test::requireThatFieldCacheRepoCanReturnDefaultFieldCache() } -Test::Test() +Fixture::Fixture() : _summaryCfg(), _resultCfg(), _markupFields() @@ -1245,33 +1195,8 @@ Test::Test() } } -Test::~Test() = default; - -int -Test::Main() -{ - TEST_INIT("docsummary_test"); - - if (_argc > 0) { - DummyFileHeaderContext::setCreator(_argv[0]); - } - TEST_DO(requireThatSummaryAdapterHandlesPutAndRemove()); - TEST_DO(requireThatAdapterHandlesAllFieldTypes()); - TEST_DO(requireThatAdapterHandlesMultipleDocuments()); - TEST_DO(requireThatAdapterHandlesDocumentIdField()); - TEST_DO(requireThatDocsumRequestIsProcessed()); - TEST_DO(requireThatRewritersAreUsed()); - TEST_DO(requireThatAttributesAreUsed()); - TEST_DO(requireThatAnnotationsAreUsed()); - TEST_DO(requireThatUrisAreUsed()); - TEST_DO(requireThatPositionsAreUsed()); - TEST_DO(requireThatRawFieldsWorks()); - TEST_DO(requireThatFieldCacheRepoCanReturnDefaultFieldCache()); - TEST_DO(requireThatSummariesTimeout()); - - TEST_DONE(); -} +Fixture::~Fixture() = default; } -TEST_APPHOOK(proton::Test); +TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index c705010ada7..aebe5a61198 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -11,7 +11,6 @@ #include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> -#include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h> #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h> #include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> @@ -19,6 +18,7 @@ #include <vespa/searchcore/proton/server/emptysearchview.h> #include <vespa/searchcore/proton/server/fast_access_document_retriever.h> #include <vespa/searchcore/proton/server/i_document_subdb_owner.h> +#include <vespa/searchcore/proton/server/igetserialnum.h> #include <vespa/searchcore/proton/server/minimal_document_retriever.h> #include <vespa/searchcore/proton/server/searchabledocsubdb.h> #include <vespa/searchcore/proton/matching/querylimiter.h> @@ -28,7 +28,6 @@ #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/test/directory_handler.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/lambdatask.h> diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 44058d48d1e..04a4edc57a6 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -511,7 +511,6 @@ struct MyConfigStore : ConfigStore { struct ReplayTransactionLogContext { - IIndexWriter::SP iwriter; MyConfigStore config_store; DocumentDBConfig::SP cfgSnap; }; diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp index edb4250ce76..2df7c5d629d 100644 --- a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp +++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp @@ -11,7 +11,7 @@ #include <vespa/vespalib/util/stringfmt.h> using search::index::DummyFileHeaderContext; -using search::transactionlog::TransLogClient; +using search::transactionlog::client::TransLogClient; using search::transactionlog::TransLogServer; using proton::DocTypeName; using proton::ProtonDiskLayout; diff --git a/searchcore/src/vespa/searchcore/config/onnx-models.def b/searchcore/src/vespa/searchcore/config/onnx-models.def index ffe71ff70c2..8d1291fa61e 100644 --- a/searchcore/src/vespa/searchcore/config/onnx-models.def +++ b/searchcore/src/vespa/searchcore/config/onnx-models.def @@ -1,5 +1,9 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.config.search.core -model[].name string -model[].fileref file +model[].name string +model[].fileref file +model[].input[].name string +model[].input[].source string +model[].output[].name string +model[].output[].as string diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp index 5f057bbd7dc..246caa6cd35 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp @@ -23,7 +23,7 @@ std::vector<vespalib::string> AttributeDiskLayout::listAttributes() { std::vector<vespalib::string> attributes; - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &dir : _dirs) { attributes.emplace_back(dir.first); } @@ -46,7 +46,7 @@ AttributeDiskLayout::scanDir() std::shared_ptr<AttributeDirectory> AttributeDiskLayout::getAttributeDir(const vespalib::string &name) { - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); auto itr = _dirs.find(name); if (itr == _dirs.end()) { return std::shared_ptr<AttributeDirectory>(); @@ -58,7 +58,7 @@ AttributeDiskLayout::getAttributeDir(const vespalib::string &name) std::shared_ptr<AttributeDirectory> AttributeDiskLayout::createAttributeDir(const vespalib::string &name) { - std::lock_guard<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_mutex> guard(_mutex); auto itr = _dirs.find(name); if (itr == _dirs.end()) { auto dir = std::make_shared<AttributeDirectory>(shared_from_this(), name); @@ -80,7 +80,7 @@ AttributeDiskLayout::removeAttributeDir(const vespalib::string &name, search::Se writer->invalidateOldSnapshots(serialNum); writer->removeInvalidSnapshots(); if (writer->removeDiskDir()) { - std::lock_guard<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_mutex> guard(_mutex); auto itr = _dirs.find(name); assert(itr != _dirs.end()); assert(dir.get() == itr->second.get()); @@ -88,7 +88,7 @@ AttributeDiskLayout::removeAttributeDir(const vespalib::string &name, search::Se writer->detach(); } } else { - std::lock_guard<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_mutex> guard(_mutex); auto itr = _dirs.find(name); if (itr != _dirs.end()) { assert(dir.get() != itr->second.get()); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h index 9e081f601bf..74cf0bb8cf8 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h @@ -19,7 +19,7 @@ class AttributeDiskLayout : public std::enable_shared_from_this<AttributeDiskLay { private: const vespalib::string _baseDir; - mutable std::shared_timed_mutex _mutex; + mutable std::shared_mutex _mutex; std::map<vespalib::string, std::shared_ptr<AttributeDirectory>> _dirs; void scanDir(); diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp index d61eaa2c6ab..251989cd263 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "feedtoken.h" +#include <vespa/persistence/spi/result.h> namespace proton::feedtoken { @@ -27,6 +28,12 @@ State::ack() } void +State::setResult(ResultUP result, bool documentWasFound) { + _documentWasFound = documentWasFound; + _result = std::move(result); +} + +void State::fail() { bool alreadySent = _alreadySent.exchange(true); diff --git a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h index cf6c1f5a7e9..411dd256ccd 100644 --- a/searchcore/src/vespa/searchcore/proton/common/feedtoken.h +++ b/searchcore/src/vespa/searchcore/proton/common/feedtoken.h @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <atomic> +namespace storage::spi { class Result; } namespace proton { typedef std::unique_ptr<storage::spi::Result> ResultUP; @@ -33,10 +33,7 @@ public: State(ITransport & transport); ~State() override; void fail(); - void setResult(ResultUP result, bool documentWasFound) { - _documentWasFound = documentWasFound; - _result = std::move(result); - } + void setResult(ResultUP result, bool documentWasFound); const storage::spi::Result &getResult() { return *_result; } protected: void ack(); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 55151e23da7..1ac313f2718 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -230,7 +230,7 @@ PersistenceEngine::removeHandler(const WriteGuard &, document::BucketSpace bucke Result PersistenceEngine::initialize() { - std::unique_lock<std::shared_timed_mutex> wguard(getWLock()); + WriteGuard wguard(getWLock()); LOG(debug, "Begin initializing persistence handlers"); HandlerSnapshot snap = getHandlerSnapshot(wguard); for (; snap.handlers().valid(); snap.handlers().next()) { @@ -255,7 +255,7 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const { // Runs in SPI thread. // No handover to write threads in persistence handlers. - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); if (id != 0) { BucketIdListResult::List emptyList; return BucketIdListResult(emptyList); @@ -273,7 +273,7 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const Result PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState &calc) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); saveClusterState(bucketSpace, calc); HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); GenericResultHandler resultHandler(snap.size()); @@ -291,7 +291,7 @@ Result PersistenceEngine::setActiveState(const Bucket& bucket, storage::spi::BucketInfo::ActiveState newState) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); HandlerSnapshot snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); GenericResultHandler resultHandler(snap.size()); for (; snap.handlers().valid(); snap.handlers().next()) { @@ -308,7 +308,7 @@ PersistenceEngine::getBucketInfo(const Bucket& b) const { // Runs in SPI thread. // No handover to write threads in persistence handlers. - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); BucketInfoResultHandler resultHandler; for (; snap.handlers().valid(); snap.handlers().next()) { @@ -330,7 +330,7 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do doc->getId().toString().c_str(), state.message().c_str()))); } } - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); DocTypeName docType(doc->getType()); LOG(spam, "putAsync(%s, %" PRIu64 ", (\"%s\", \"%s\"))", bucket.toString().c_str(), static_cast<uint64_t>(ts.getValue()), docType.toString().c_str(), doc->getId().toString().c_str()); @@ -350,7 +350,7 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do void PersistenceEngine::removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context&, OperationComplete::UP onComplete) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); LOG(spam, "remove(%s, %" PRIu64 ", \"%s\")", b.toString().c_str(), static_cast<uint64_t>(t.getValue()), did.toString().c_str()); if (!did.hasDocType()) { @@ -397,7 +397,7 @@ PersistenceEngine::updateAsync(const Bucket& b, Timestamp t, DocumentUpdate::SP upd->getType().getName().c_str(), e.getMessage().c_str()))); } - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); DocTypeName docType(upd->getType()); LOG(spam, "update(%s, %" PRIu64 ", (\"%s\", \"%s\"), createIfNonExistent='%s')", b.toString().c_str(), static_cast<uint64_t>(t.getValue()), docType.toString().c_str(), @@ -423,7 +423,7 @@ PersistenceEngine::updateAsync(const Bucket& b, Timestamp t, DocumentUpdate::SP PersistenceEngine::GetResult PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const DocumentId& did, Context& context) const { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); HandlerSnapshot snapshot = getHandlerSnapshot(rguard, b.getBucketSpace()); for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { @@ -455,7 +455,7 @@ PersistenceEngine::CreateIteratorResult PersistenceEngine::createIterator(const Bucket &bucket, FieldSetSP fields, const Selection &selection, IncludedVersions versions, Context &context) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); HandlerSnapshot snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace()); auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, std::move(fields), selection, @@ -481,7 +481,7 @@ PersistenceEngine::createIterator(const Bucket &bucket, FieldSetSP fields, const PersistenceEngine::IterateResult PersistenceEngine::iterate(IteratorId id, uint64_t maxByteSize, Context&) const { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); IteratorEntry *iteratorEntry; { std::lock_guard<std::mutex> guard(_iterators_lock); @@ -515,7 +515,7 @@ PersistenceEngine::iterate(IteratorId id, uint64_t maxByteSize, Context&) const Result PersistenceEngine::destroyIterator(IteratorId id, Context&) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); std::lock_guard<std::mutex> guard(_iterators_lock); auto it = _iterators.find(id); if (it == _iterators.end()) { @@ -533,7 +533,7 @@ PersistenceEngine::destroyIterator(IteratorId id, Context&) Result PersistenceEngine::createBucket(const Bucket &b, Context &) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); LOG(spam, "createBucket(%s)", b.toString().c_str()); HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); TransportLatch latch(snap.size()); @@ -549,7 +549,7 @@ PersistenceEngine::createBucket(const Bucket &b, Context &) Result PersistenceEngine::deleteBucket(const Bucket& b, Context&) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); LOG(spam, "deleteBucket(%s)", b.toString().c_str()); HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); TransportLatch latch(snap.size()); @@ -565,7 +565,7 @@ PersistenceEngine::deleteBucket(const Bucket& b, Context&) BucketIdListResult PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); typedef BucketIdListResultV MBV; MBV extraModifiedBuckets; { @@ -589,7 +589,7 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const Result PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); LOG(spam, "split(%s, %s, %s)", source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str()); assert(source.getBucketSpace() == target1.getBucketSpace()); assert(source.getBucketSpace() == target2.getBucketSpace()); @@ -607,7 +607,7 @@ PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Buck Result PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) { - std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + ReadGuard rguard(_rwMutex); LOG(spam, "join(%s, %s, %s)", source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str()); assert(source1.getBucketSpace() == target.getBucketSpace()); assert(source2.getBucketSpace() == target.getBucketSpace()); @@ -740,10 +740,10 @@ PersistenceEngine::populateInitialBucketDB(const WriteGuard & guard, BucketSpace trHandler.await(); } -std::unique_lock<std::shared_timed_mutex> +std::unique_lock<std::shared_mutex> PersistenceEngine::getWLock() const { - return std::unique_lock<std::shared_timed_mutex>(_rwMutex); + return WriteGuard(_rwMutex); } } // storage diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index a874d91eb20..9ee42823f9c 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -29,7 +29,6 @@ private: using IncludedVersions = storage::spi::IncludedVersions; using IterateResult = storage::spi::IterateResult; using IteratorId = storage::spi::IteratorId; - using MaintenanceLevel = storage::spi::MaintenanceLevel; using PartitionId = storage::spi::PartitionId; using PartitionStateListResult = storage::spi::PartitionStateListResult; using RemoveResult = storage::spi::RemoveResult; @@ -71,10 +70,10 @@ private: const IResourceWriteFilter &_writeFilter; std::unordered_map<BucketSpace, ClusterState::SP, BucketSpace::hash> _clusterStates; mutable ExtraModifiedBuckets _extraModifiedBuckets; - mutable std::shared_timed_mutex _rwMutex; + mutable std::shared_mutex _rwMutex; - using ReadGuard = std::shared_lock<std::shared_timed_mutex>; - using WriteGuard = std::unique_lock<std::shared_timed_mutex>; + using ReadGuard = std::shared_lock<std::shared_mutex>; + using WriteGuard = std::unique_lock<std::shared_mutex>; IPersistenceHandler * getHandler(const ReadGuard & guard, document::BucketSpace bucketSpace, const DocTypeName &docType) const; HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard) const; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h index 60ff696cb03..6712a35bd65 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resulthandler.h @@ -8,7 +8,7 @@ namespace proton { template <typename ResultType> class IResultHandler { public: - virtual ~IResultHandler() { } + virtual ~IResultHandler() = default; virtual void handle(const ResultType &result) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h index da4e19d3584..0e6b5d23053 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/persistence/spi/result.h> +#include <vespa/persistence/spi/operationcomplete.h> #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/vespalib/util/sequence.h> #include <vespa/vespalib/util/count_down_latch.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 0a3c1015d3a..e81bb3c0037 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -100,7 +100,6 @@ vespa_add_library(searchcore_server STATIC storeonlyfeedview.cpp summaryadapter.cpp threading_service_config.cpp - tlcproxy.cpp tlssyncer.cpp transactionlogmanager.cpp transactionlogmanagerbase.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index 5625d41ccdc..079b8699ce4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -6,6 +6,8 @@ #include "replaypacketdispatcher.h" #include "ibucketstatecalculator.h" #include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/document/bucket/bucketspace.h> +#include <vespa/document/base/globalid.h> #include <vespa/searchlib/common/serialnum.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 9cdc70d6627..46bcb0e49bb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -11,6 +11,7 @@ #include "lid_space_compaction_handler.h" #include "maintenance_jobs_injector.h" #include "reconfig_params.h" +#include "feedhandler.h" #include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> @@ -119,7 +120,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, IDocumentDBOwner &owner, vespalib::SyncableThreadExecutor &warmupExecutor, vespalib::ThreadStackExecutorBase &sharedExecutor, - search::transactionlog::Writer &tlsDirectWriter, + const search::transactionlog::WriterFactory &tlsWriterFactory, MetricsWireService &metricsWireService, const FileHeaderContext &fileHeaderContext, ConfigStore::UP config_store, @@ -166,9 +167,9 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _dmUsageForwarder(_writeService.master()), _writeFilter(), _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()), - _feedHandler(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsDirectWriter), - _visibility(_feedHandler, _writeService, _feedView), - _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, + _feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)), + _visibility(*_feedHandler, _writeService, _feedView), + _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, makeSubDBConfig(protonCfg.distribution, findDocumentDB(protonCfg.documentdb, docTypeName.getName())->allocation, @@ -184,11 +185,11 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, LOG(debug, "DocumentDB(%s): Creating database in directory '%s'", _docTypeName.toString().c_str(), _baseDir.c_str()); - _feedHandler.init(_config_store->getOldestSerialNum()); - _feedHandler.setBucketDBHandler(&_subDBs.getBucketDBHandler()); + _feedHandler->init(_config_store->getOldestSerialNum()); + _feedHandler->setBucketDBHandler(&_subDBs.getBucketDBHandler()); saveInitialConfig(*configSnapshot); resumeSaveConfig(); - SerialNum configSerial = _config_store->getPrevValidSerial(_feedHandler.getPrunedSerialNum() + 1); + SerialNum configSerial = _config_store->getPrevValidSerial(_feedHandler->getPrunedSerialNum() + 1); assert(configSerial > 0); DocumentDBConfig::SP loaded_config; _config_store->loadConfig(*configSnapshot, configSerial, loaded_config); @@ -348,11 +349,11 @@ DocumentDB::enterReprocessState() if (!runner.empty()) { runner.run(); NoopOperation op; - _feedHandler.storeOperationSync(op); + _feedHandler->storeOperationSync(op); sync(op.getSerialNum()); _subDBs.pruneRemovedFields(op.getSerialNum()); } - _subDBs.onReprocessDone(_feedHandler.getSerialNum()); + _subDBs.onReprocessDone(_feedHandler->getSerialNum()); enterOnlineState(); } @@ -362,12 +363,12 @@ DocumentDB::enterOnlineState() { // Called by executor thread // Ensure that all replayed operations are committed to memory structures - _feedView.get()->forceCommit(_feedHandler.getSerialNum()); + _feedView.get()->forceCommit(_feedHandler->getSerialNum()); _writeService.sync(); (void) _state.enterOnlineState(); // Consider delayed pruning of transaction log and config history - _feedHandler.considerDelayedPrune(); + _feedHandler->considerDelayedPrune(); performStartMaintenance(); } @@ -434,14 +435,14 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum bool equalReplayConfig = *DocumentDBConfig::makeReplayConfig(configSnapshot) == *DocumentDBConfig::makeReplayConfig(_activeConfigSnapshot); - bool tlsReplayDone = _feedHandler.getTransactionLogReplayDone(); + bool tlsReplayDone = _feedHandler->getTransactionLogReplayDone(); if (!equalReplayConfig && tlsReplayDone) { - sync(_feedHandler.getSerialNum()); - serialNum = _feedHandler.incSerialNum(); + sync(_feedHandler->getSerialNum()); + serialNum = _feedHandler->incSerialNum(); _config_store->saveConfig(*configSnapshot, serialNum); // save entry in transaction log NewConfigOperation op(serialNum, *_config_store); - _feedHandler.storeOperationSync(op); + _feedHandler->storeOperationSync(op); sync(op.getSerialNum()); } bool hasVisibilityDelayChanged = false; @@ -461,7 +462,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum } if (params.shouldSubDbsChange() || hasVisibilityDelayChanged) { applySubDBConfig(*configSnapshot, serialNum, params); - if (serialNum < _feedHandler.getSerialNum()) { + if (serialNum < _feedHandler->getSerialNum()) { // Not last entry in tls. Reprocessing should already be done. _subDBs.getReprocessingRunner().reset(); } @@ -577,7 +578,7 @@ DocumentDB::close() // What about queued tasks ? _writeService.shutdown(); _maintenanceController.kill(); - _feedHandler.close(); + _feedHandler->close(); // Assumes that feed engine has been closed. If only this document DB // is going away while system is still up and running then caller must // ensure that routing has been torn down and pending messages have been @@ -625,16 +626,16 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot) if (_config_store->getBestSerialNum() != 0) return; // Initial config already present - SerialNum confSerial = _feedHandler.incSerialNum(); + SerialNum confSerial = _feedHandler->incSerialNum(); // Elide save of new config entry in transaction log, it would be // pruned at once anyway. // save noop entry in transaction log NoopOperation op; - _feedHandler.storeOperationSync(op); + _feedHandler->storeOperationSync(op); sync(op.getSerialNum()); // Wipe everything in transaction log before initial config. try { - _feedHandler.tlsPrune(confSerial); // throws on error + _feedHandler->tlsPrune(confSerial); // throws on error } catch (const vespalib::IllegalStateException & e) { LOG(warning, "DocumentDB(%s): saveInitialConfig() failed pruning due to '%s'", _docTypeName.toString().c_str(), e.what()); @@ -648,13 +649,13 @@ DocumentDB::resumeSaveConfig() SerialNum bestSerial = _config_store->getBestSerialNum(); if (bestSerial == 0) return; - if (bestSerial != _feedHandler.getSerialNum() + 1) + if (bestSerial != _feedHandler->getSerialNum() + 1) return; // proton was interrupted when saving later config. - SerialNum confSerial = _feedHandler.incSerialNum(); + SerialNum confSerial = _feedHandler->incSerialNum(); // resume operation, i.e. save config entry in transaction log NewConfigOperation op(confSerial, *_config_store); - _feedHandler.storeOperationSync(op); + _feedHandler->storeOperationSync(op); sync(op.getSerialNum()); } @@ -671,9 +672,9 @@ DocumentDB::onTransactionLogReplayDone() } if (_validateAndSanitizeDocStore) { LOG(info, "Validating documentdb %s", getName().c_str()); - SerialNum serialNum = _feedHandler.getSerialNum(); + SerialNum serialNum = _feedHandler->getSerialNum(); sync(serialNum); - _subDBs.validateDocStore(_feedHandler, serialNum); + _subDBs.validateDocStore(*_feedHandler, serialNum); } } @@ -720,7 +721,7 @@ DocumentDB::startTransactionLogReplay() SerialNum oldestFlushedSerial = getOldestFlushedSerial(); SerialNum newestFlushedSerial = getNewestFlushedSerial(); (void) _state.enterReplayTransactionLogState(); - _feedHandler.replayTransactionLog(readySubDB->getIndexManager()-> + _feedHandler->replayTransactionLog(readySubDB->getIndexManager()-> getFlushedSerialNum(), readySubDB->getSummaryManager()-> getBackingStore().lastSyncToken(), @@ -780,7 +781,7 @@ DocumentDB::getFlushTargets() void DocumentDB::flushDone(SerialNum flushedSerial) { - _feedHandler.flushDone(flushedSerial); + _feedHandler->flushDone(flushedSerial); } void @@ -813,9 +814,9 @@ DocumentDB::enterRedoReprocessState() return; } runner.run(); - _subDBs.onReprocessDone(_feedHandler.getSerialNum()); + _subDBs.onReprocessDone(_feedHandler->getSerialNum()); NoopOperation op; - _feedHandler.storeOperationSync(op); + _feedHandler->storeOperationSync(op); sync(op.getSerialNum()); _subDBs.pruneRemovedFields(op.getSerialNum()); } @@ -850,8 +851,8 @@ DocumentDB::reportStatus() const if (_initGate.getCount() != 0) { return StatusReport::create(params.state(StatusReport::PARTIAL). message("DocumentDB initializing components")); - } else if (_feedHandler.isDoingReplay()) { - float progress = _feedHandler.getReplayProgress() * 100.0f; + } else if (_feedHandler->isDoingReplay()) { + float progress = _feedHandler->getReplayProgress() * 100.0f; vespalib::string msg = vespalib::make_string("DocumentDB replay transaction log on startup (%u%% done)", static_cast<uint32_t>(progress)); return StatusReport::create(params.state(StatusReport::PARTIAL).progress(progress).message(msg)); @@ -914,7 +915,7 @@ DocumentDB::syncFeedView() _writeService.sync(); _feedView.set(newFeedView); - _feedHandler.setActiveFeedView(newFeedView.get()); + _feedHandler->setActiveFeedView(newFeedView.get()); _subDBs.createRetrievers(); _subDBs.maintenanceSync(_maintenanceController, _visibility); @@ -936,16 +937,16 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std _maintenanceController.killJobs(); MaintenanceJobsInjector::injectJobs(_maintenanceController, config, - _feedHandler, // IHeartBeatHandler + *_feedHandler, // IHeartBeatHandler *_sessionManager, // ISessionCachePruner _lidSpaceCompactionHandlers, - _feedHandler, // IOperationStorer + *_feedHandler, // IOperationStorer _maintenanceController, // IFrozenBucketHandler _subDBs.getBucketCreateNotifier(), _docTypeName.getName(), _bucketSpace, - _feedHandler, // IPruneRemovedDocumentsHandler - _feedHandler, // IDocumentMoveHandler + *_feedHandler, // IPruneRemovedDocumentsHandler + *_feedHandler, // IDocumentMoveHandler _clusterStateHandler, // IBucketModifiedHandler _clusterStateHandler, // IClusterStateChangedNotifier _bucketHandler, // IBucketStateChangedNotifier @@ -1068,7 +1069,16 @@ void DocumentDB::sync(SerialNum syncTo) { LOG(spam, "DocumentDB(%s): sync(): serialNum=%" PRIu64, _docTypeName.toString().c_str(), syncTo); - _feedHandler.syncTls(syncTo); + _feedHandler->syncTls(syncTo); +} + +SerialNum +DocumentDB::getCurrentSerialNumber() const +{ + // Called by flush scheduler thread, by executor task or + // visitor callback. + // XXX: Contains future value during replay. + return _feedHandler->getSerialNum(); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 0c93dc88ada..ef0ed729622 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -11,7 +11,6 @@ #include "documentdbconfig.h" #include "documentsubdbcollection.h" #include "executorthreadingservice.h" -#include "feedhandler.h" #include "i_document_subdb_owner.h" #include "i_feed_handler_owner.h" #include "i_lid_space_compaction_handler.h" @@ -38,7 +37,10 @@ namespace search { namespace common { class FileHeaderContext; } - namespace transactionlog { class TransLogClient; } + namespace transactionlog { + class TransLogClient; + class WriterFactory; + } } namespace vespa::config::search::core::internal { class InternalProtonType; } @@ -136,7 +138,7 @@ private: DiskMemUsageForwarder _dmUsageForwarder; AttributeUsageFilter _writeFilter; std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; - FeedHandler _feedHandler; + std::unique_ptr<FeedHandler> _feedHandler; VisibilityHandler _visibility; DocumentSubDBCollection _subDBs; MaintenanceController _maintenanceController; @@ -251,7 +253,7 @@ public: IDocumentDBOwner &owner, vespalib::SyncableThreadExecutor &warmupExecutor, vespalib::ThreadStackExecutorBase &sharedExecutor, - search::transactionlog::Writer &tlsDirectWriter, + const search::transactionlog::WriterFactory &tlsWriterFactory, MetricsWireService &metricsWireService, const search::common::FileHeaderContext &fileHeaderContext, ConfigStore::UP config_store, @@ -338,7 +340,7 @@ public: /** * Returns the feed handler for this database. */ - FeedHandler & getFeedHandler() { return _feedHandler; } + FeedHandler & getFeedHandler() { return *_feedHandler; } /** * Returns the bucket handler for this database. @@ -371,24 +373,14 @@ public: virtual SerialNum getNewestFlushedSerial(); std::unique_ptr<search::engine::SearchReply> - match(const search::engine::SearchRequest &req, - vespalib::ThreadBundle &threadBundle) const; + match(const search::engine::SearchRequest &req, vespalib::ThreadBundle &threadBundle) const; std::unique_ptr<search::engine::DocsumReply> getDocsums(const search::engine::DocsumRequest & request); IFlushTargetList getFlushTargets(); void flushDone(SerialNum flushedSerial); - - virtual SerialNum - getCurrentSerialNumber() const - { - // Called by flush scheduler thread, by executor task or - // visitor callback. - // XXX: Contains future value during replay. - return _feedHandler.getSerialNum(); - } - + virtual SerialNum getCurrentSerialNumber() const; StatusReportUP reportStatus() const; /** @@ -438,6 +430,7 @@ public: void waitForOnlineState(); IDiskMemUsageListener *diskMemUsageListener() { return &_dmUsageForwarder; } std::shared_ptr<const ITransientMemoryUsageProvider> transient_memory_usage_provider(); + ExecutorThreadingService & getWriteService() { return _writeService; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 684bf0125ac..3ab5d1ca065 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -5,12 +5,12 @@ #include "feedstates.h" #include "i_feed_handler_owner.h" #include "ifeedview.h" -#include "tlcproxy.h" #include "configstore.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/searchcore/proton/persistenceengine/transport_latch.h> @@ -18,6 +18,7 @@ #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/gatecallback.h> +#include <vespa/searchlib/transactionlog/client_session.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> #include <unistd.h> @@ -45,22 +46,45 @@ namespace proton { namespace { +using search::SerialNum; + bool ignoreOperation(const DocumentOperation &op) { return (op.getPrevTimestamp() != 0) && (op.getTimestamp() < op.getPrevTimestamp()); } -} // namespace - -void FeedHandler::TlsMgrWriter::storeOperation(const FeedOperation &op, DoneCallback onDone) { - TlcProxy(_tls_mgr.getDomainName(), *_tlsDirectWriter).storeOperation(op, std::move(onDone)); -} -bool FeedHandler::TlsMgrWriter::erase(SerialNum oldest_to_keep) { +class TlsMgrWriter : public TlsWriter { + TransactionLogManager &_tls_mgr; + std::shared_ptr<search::transactionlog::Writer> _writer; +public: + TlsMgrWriter(TransactionLogManager &tls_mgr, + const search::transactionlog::WriterFactory & factory) : + _tls_mgr(tls_mgr), + _writer(factory.getWriter(tls_mgr.getDomainName())) + { } + void storeOperation(const FeedOperation &op, DoneCallback onDone) override; + bool erase(SerialNum oldest_to_keep) override; + SerialNum sync(SerialNum syncTo) override; +}; + + +void TlsMgrWriter::storeOperation(const FeedOperation &op, DoneCallback onDone) { + using Packet = search::transactionlog::Packet; + vespalib::nbostream stream; + op.serialize(stream); + LOG(debug, "storeOperation(): serialNum(%" PRIu64 "), type(%u), size(%zu)", + op.getSerialNum(), (uint32_t)op.getType(), stream.size()); + Packet::Entry entry(op.getSerialNum(), op.getType(), vespalib::ConstBufferRef(stream.data(), stream.size())); + Packet packet(entry.serializedSize()); + packet.add(entry); + _writer->commit(packet, std::move(onDone)); +} +bool TlsMgrWriter::erase(SerialNum oldest_to_keep) { return _tls_mgr.getSession()->erase(oldest_to_keep); } -search::SerialNum -FeedHandler::TlsMgrWriter::sync(SerialNum syncTo) +SerialNum +TlsMgrWriter::sync(SerialNum syncTo) { for (int retryCount = 0; retryCount < 10; ++retryCount) { SerialNum syncedTo(0); @@ -80,11 +104,13 @@ FeedHandler::TlsMgrWriter::sync(SerialNum syncTo) throw IllegalStateException(make_string("Failed to sync TLS to token %" PRIu64 ".", syncTo)); } +} // namespace + void FeedHandler::doHandleOperation(FeedToken token, FeedOperation::UP op) { assert(_writeService.master().isCurrentThread()); - std::lock_guard<std::mutex> guard(_feedLock); + // Since _feedState is only modified in the master thread we can skip the lock here. _feedState->handleOperation(std::move(token), std::move(op)); } @@ -296,43 +322,41 @@ FeedHandler::considerDelayedPrune() } -FeedState::SP +std::shared_ptr<FeedState> FeedHandler::getFeedState() const { - FeedState::SP state; - { - std::lock_guard<std::mutex> guard(_feedLock); - state = _feedState; - } - return state; + ReadGuard guard(_feedLock); + return _feedState; } - void -FeedHandler::changeFeedState(FeedState::SP newState) +FeedHandler::changeFeedState(FeedStateSP newState) { - std::lock_guard<std::mutex> guard(_feedLock); - changeFeedState(std::move(newState), guard); + if (_writeService.master().isCurrentThread()) { + doChangeFeedState(std::move(newState)); + } else { + _writeService.master().execute(makeLambdaTask([this, newState=std::move(newState)] () { doChangeFeedState(std::move(newState));})); + _writeService.master().sync(); + } } - void -FeedHandler::changeFeedState(FeedState::SP newState, const std::lock_guard<std::mutex> &) +FeedHandler::doChangeFeedState(FeedStateSP newState) { + WriteGuard guard(_feedLock); LOG(debug, "Change feed state from '%s' -> '%s'", _feedState->getName().c_str(), newState->getName().c_str()); _feedState = std::move(newState); } - FeedHandler::FeedHandler(IThreadingService &writeService, const vespalib::string &tlsSpec, const DocTypeName &docTypeName, IFeedHandlerOwner &owner, const IResourceWriteFilter &writeFilter, IReplayConfig &replayConfig, - search::transactionlog::Writer & tlsDirectWriter, + const TlsWriterFactory & tlsWriterFactory, TlsWriter * tlsWriter) - : search::transactionlog::TransLogClient::Session::Callback(), + : search::transactionlog::client::Callback(), IDocumentMoveHandler(), IPruneRemovedDocumentsHandler(), IHeartBeatHandler(), @@ -344,8 +368,9 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _writeFilter(writeFilter), _replayConfig(replayConfig), _tlsMgr(tlsSpec, docTypeName.getName()), - _tlsMgrWriter(_tlsMgr, &tlsDirectWriter), - _tlsWriter(tlsWriter ? *tlsWriter : _tlsMgrWriter), + _tlsWriterfactory(tlsWriterFactory), + _tlsMgrWriter(), + _tlsWriter(tlsWriter), _tlsReplayProgress(), _serialNum(0), _prunedSerialNum(0), @@ -369,6 +394,10 @@ void FeedHandler::init(SerialNum oldestConfigSerial) { _tlsMgr.init(oldestConfigSerial, _prunedSerialNum, _serialNum); + if (_tlsWriter == nullptr) { + _tlsMgrWriter = std::make_unique<TlsMgrWriter>(_tlsMgr, _tlsWriterfactory); + _tlsWriter = _tlsMgrWriter.get(); + } _allowSync = true; syncTls(_serialNum); } @@ -442,7 +471,7 @@ FeedHandler::storeOperation(const FeedOperation &op, TlsWriter::DoneCallback onD if (!op.getSerialNum()) { const_cast<FeedOperation &>(op).setSerialNum(incSerialNum()); } - _tlsWriter.storeOperation(op, std::move(onDone)); + _tlsWriter->storeOperation(op, std::move(onDone)); } void @@ -454,7 +483,7 @@ FeedHandler::storeOperationSync(const FeedOperation &op) { void FeedHandler::tlsPrune(SerialNum oldest_to_keep) { - if (!_tlsWriter.erase(oldest_to_keep)) { + if (!_tlsWriter->erase(oldest_to_keep)) { throw IllegalStateException(make_string("Failed to prune TLS to token %" PRIu64 ".", oldest_to_keep)); } _prunedSerialNum = oldest_to_keep; @@ -630,7 +659,7 @@ FeedHandler::receive(const Packet &packet) // Called directly when replaying transaction log // (by fnet thread). Called via DocumentDB::recoverPacket() when // recovering from another node. - FeedState::SP state = getFeedState(); + FeedStateSP state = getFeedState(); auto wrap = make_shared<PacketWrapper>(packet, _tlsReplayProgress.get()); state->receive(wrap, _writeService.master()); wrap->gate.await(); @@ -666,7 +695,7 @@ FeedHandler::syncTls(SerialNum syncTo) if (!_allowSync) { throw IllegalStateException(make_string("Attempted to sync TLS to token %" PRIu64 " at wrong time.", syncTo)); } - SerialNum syncedTo(_tlsWriter.sync(syncTo)); + SerialNum syncedTo(_tlsWriter->sync(syncTo)); { std::lock_guard<std::mutex> guard(_syncLock); if (_syncedSerialNum < syncedTo) diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 2e6b2616118..823361f8737 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -10,12 +10,14 @@ #include "tlswriter.h" #include "transactionlogmanager.h" #include <persistence/spi/types.h> +#include <vespa/document/bucket/bucketid.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/feedtoken.h> -#include <vespa/searchlib/transactionlog/translogclient.h> -#include <mutex> +#include <vespa/searchlib/transactionlog/client_common.h> +#include <shared_mutex> -namespace searchcorespi { namespace index { struct IThreadingService; } } +namespace searchcorespi::index { struct IThreadingService; } +namespace document { class DocumentTypeRepo; } namespace proton { struct ConfigStore; @@ -41,7 +43,7 @@ namespace bucketdb { class IBucketDBHandler; } * Class handling all aspects of feeding for a document database. * In addition to regular feeding this also includes handling the transaction log. */ -class FeedHandler: private search::transactionlog::TransLogClient::Session::Callback, +class FeedHandler: private search::transactionlog::client::Callback, public IDocumentMoveHandler, public IPruneRemovedDocumentsHandler, public IHeartBeatHandler, @@ -49,28 +51,17 @@ class FeedHandler: private search::transactionlog::TransLogClient::Session::Call public IGetSerialNum { private: - typedef search::transactionlog::Packet Packet; - typedef search::transactionlog::RPC RPC; - typedef search::SerialNum SerialNum; - typedef storage::spi::Timestamp Timestamp; - typedef document::BucketId BucketId; + using Packet = search::transactionlog::Packet; + using RPC = search::transactionlog::client::RPC; + using SerialNum = search::SerialNum; + using Timestamp = storage::spi::Timestamp; + using BucketId = document::BucketId; using FeedStateSP = std::shared_ptr<FeedState>; using FeedOperationUP = std::unique_ptr<FeedOperation>; - - class TlsMgrWriter : public TlsWriter { - TransactionLogManager &_tls_mgr; - search::transactionlog::Writer *_tlsDirectWriter; - public: - TlsMgrWriter(TransactionLogManager &tls_mgr, - search::transactionlog::Writer * tlsDirectWriter) : - _tls_mgr(tls_mgr), - _tlsDirectWriter(tlsDirectWriter) - { } - void storeOperation(const FeedOperation &op, DoneCallback onDone) override; - bool erase(SerialNum oldest_to_keep) override; - SerialNum sync(SerialNum syncTo) override; - }; - typedef searchcorespi::index::IThreadingService IThreadingService; + using ReadGuard = std::shared_lock<std::shared_mutex>; + using WriteGuard = std::unique_lock<std::shared_mutex>; + using IThreadingService = searchcorespi::index::IThreadingService; + using TlsWriterFactory = search::transactionlog::WriterFactory; IThreadingService &_writeService; DocTypeName _docTypeName; @@ -78,14 +69,15 @@ private: const IResourceWriteFilter &_writeFilter; IReplayConfig &_replayConfig; TransactionLogManager _tlsMgr; - TlsMgrWriter _tlsMgrWriter; - TlsWriter &_tlsWriter; + const TlsWriterFactory &_tlsWriterfactory; + std::unique_ptr<TlsWriter> _tlsMgrWriter; + TlsWriter *_tlsWriter; TlsReplayProgress::UP _tlsReplayProgress; // the serial num of the last message in the transaction log SerialNum _serialNum; SerialNum _prunedSerialNum; bool _delayedPrune; - mutable std::mutex _feedLock; + mutable std::shared_mutex _feedLock; FeedStateSP _feedState; // used by master write thread tasks IFeedView *_activeFeedView; @@ -132,7 +124,7 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); - void changeFeedState(FeedStateSP newState, const std::lock_guard<std::mutex> &feedGuard); + void doChangeFeedState(FeedStateSP newState); public: FeedHandler(const FeedHandler &) = delete; FeedHandler & operator = (const FeedHandler &) = delete; @@ -152,7 +144,7 @@ public: IFeedHandlerOwner &owner, const IResourceWriteFilter &writerFilter, IReplayConfig &replayConfig, - search::transactionlog::Writer & writer, + const TlsWriterFactory & writer, TlsWriter * tlsWriter = nullptr); ~FeedHandler() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp index 3628505ed66..be09816b9a1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.cpp @@ -4,7 +4,6 @@ #include <vespa/searchcore/proton/feedoperation/feedoperation.h> #include <vespa/vespalib/util/exceptions.h> -using document::BucketId; using vespalib::IllegalStateException; using vespalib::make_string; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstate.h b/searchcore/src/vespa/searchcore/proton/server/feedstate.h index fa0a1702499..6de1d7a4322 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstate.h @@ -2,17 +2,16 @@ #pragma once -#include "packetwrapper.h" -#include <vespa/document/bucket/bucketid.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcore/proton/common/feedtoken.h> -#include <vespa/searchcore/proton/persistenceengine/resulthandler.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/executor.h> +namespace vespalib { + class Executor; +} namespace proton { class FeedOperation; +class PacketWrapper; /** * Class representing the current state of a feed handler. @@ -26,21 +25,20 @@ private: protected: using FeedOperationUP = std::unique_ptr<FeedOperation>; + using PacketWrapperSP = std::shared_ptr<PacketWrapper>; void throwExceptionInReceive(const vespalib::string &docType, uint64_t serialRangeFrom, uint64_t serialRangeTo, size_t packetSize); void throwExceptionInHandleOperation(const vespalib::string &docType, const FeedOperation &op); public: - typedef std::shared_ptr<FeedState> SP; - FeedState(Type type) : _type(type) {} - virtual ~FeedState() {} + virtual ~FeedState() = default; Type getType() const { return _type; } vespalib::string getName() const; virtual void handleOperation(FeedToken token, FeedOperationUP op) = 0; - virtual void receive(const PacketWrapper::SP &wrap, vespalib::Executor &executor) = 0; + virtual void receive(const PacketWrapperSP &wrap, vespalib::Executor &executor) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index d01c25d9c1e..5214e13fe79 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -16,7 +16,7 @@ LOG_SETUP(".proton.server.feedstates"); using search::transactionlog::Packet; -using search::transactionlog::RPC; +using search::transactionlog::client::RPC; using search::SerialNum; using vespalib::Executor; using vespalib::makeClosure; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.h b/searchcore/src/vespa/searchcore/proton/server/feedstates.h index 2cf0ee1a4dd..0c2e9109cce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.h @@ -4,6 +4,7 @@ #include "feedhandler.h" #include "feedstate.h" +#include "packetwrapper.h" #include "ireplaypackethandler.h" #include <vespa/searchcore/proton/common/commit_time_tracker.h> @@ -33,7 +34,7 @@ public: throwExceptionInHandleOperation(_doc_type_name, *op); } - void receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { + void receive(const PacketWrapperSP &wrap, vespalib::Executor &) override { throwExceptionInReceive(_doc_type_name.c_str(), wrap->packet.range().from(), wrap->packet.range().to(), wrap->packet.size()); } @@ -60,7 +61,7 @@ public: throwExceptionInHandleOperation(_doc_type_name, *op); } - void receive(const PacketWrapper::SP &wrap, vespalib::Executor &executor) override; + void receive(const PacketWrapperSP &wrap, vespalib::Executor &executor) override; }; @@ -81,7 +82,7 @@ public: _handler.performOperation(std::move(token), std::move(op)); } - void receive(const PacketWrapper::SP &wrap, vespalib::Executor &) override { + void receive(const PacketWrapperSP &wrap, vespalib::Executor &) override { throwExceptionInReceive(_handler.getDocTypeName().c_str(), wrap->packet.range().from(), wrap->packet.range().to(), wrap->packet.size()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp index d1f7a88c076..de4249c29a8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.cpp @@ -4,10 +4,7 @@ namespace proton { -IReplayConfig::~IReplayConfig() -{ -} - +IReplayConfig::~IReplayConfig() = default; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h index 0ac3b0d8a68..6b9c522b688 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/ireplayconfig.h @@ -2,19 +2,16 @@ #pragma once -#include <vespa/searchlib/transactionlog/translogclient.h> +#include <vespa/searchlib/common/serialnum.h> -namespace proton -{ +namespace proton { class IReplayConfig { public: - virtual - ~IReplayConfig(); + virtual ~IReplayConfig(); - virtual void - replayConfig(search::SerialNum serialNum) = 0; + virtual void replayConfig(search::SerialNum serialNum) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h b/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h index 6224b3b693a..c36652ec847 100644 --- a/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/server/packetwrapper.h @@ -4,6 +4,7 @@ #include "tls_replay_progress.h" #include <vespa/searchlib/transactionlog/common.h> +#include <vespa/searchlib/transactionlog/client_common.h> #include <vespa/vespalib/util/gate.h> namespace proton { @@ -16,14 +17,14 @@ struct PacketWrapper { const search::transactionlog::Packet &packet; TlsReplayProgress *progress; - search::transactionlog::RPC::Result result; + search::transactionlog::client::RPC::Result result; vespalib::Gate gate; PacketWrapper(const search::transactionlog::Packet &p, TlsReplayProgress *progress_) : packet(p), progress(progress_), - result(search::transactionlog::RPC::ERROR), + result(search::transactionlog::client::RPC::ERROR), gate() { } diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp index 7d151a9ef14..c2f80f78b8a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp @@ -2,6 +2,7 @@ #include "persistencehandlerproxy.h" #include "documentdb.h" +#include "feedhandler.h" #include <vespa/searchcore/proton/feedoperation/createbucketoperation.h> #include <vespa/searchcore/proton/feedoperation/deletebucketoperation.h> #include <vespa/searchcore/proton/feedoperation/joinbucketsoperation.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index c40f1263324..321662ea404 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -492,7 +492,7 @@ Proton::closeDocumentDBs(vespalib::ThreadStackExecutorBase & executor) { size_t Proton::getNumDocs() const { size_t numDocs(0); - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &kv : _documentDBMap) { numDocs += kv.second->getNumDocs(); } @@ -502,7 +502,7 @@ size_t Proton::getNumDocs() const size_t Proton::getNumActiveDocs() const { size_t numDocs(0); - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &kv : _documentDBMap) { numDocs += kv.second->getNumActiveDocs(); } @@ -532,7 +532,7 @@ Proton::getDelayedConfigs() const { std::ostringstream res; bool first = true; - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &kv : _documentDBMap) { if (kv.second->getDelayedConfig()) { if (!first) { @@ -549,7 +549,7 @@ StatusReport::List Proton::getStatusReports() const { StatusReport::List reports; - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); reports.push_back(StatusReport::SP(_matchEngine->reportStatus())); for (const auto &kv : _documentDBMap) { reports.push_back(StatusReport::SP(kv.second->reportStatus())); @@ -566,7 +566,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, { const ProtonConfig &config(bootstrapConfig->getProtonConfig()); - std::lock_guard<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_mutex> guard(_mutex); DocTypeName docTypeName(docType.getName()); auto it = _documentDBMap.find(docTypeName); if (it != _documentDBMap.end()) { @@ -607,7 +607,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, _documentDBMap[docTypeName] = ret; if (_persistenceEngine) { // Not allowed to get to service layer to call pause(). - std::unique_lock<std::shared_timed_mutex> persistenceWGuard(_persistenceEngine->getWLock()); + std::unique_lock<std::shared_mutex> persistenceWGuard(_persistenceEngine->getWLock()); auto persistenceHandler = std::make_shared<PersistenceHandlerProxy>(ret); if (!_isInitializing) { _persistenceEngine->propagateSavedClusterState(bucketSpace, *persistenceHandler); @@ -632,7 +632,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) { DocumentDB::SP old; { - std::lock_guard<std::shared_timed_mutex> guard(_mutex); + std::lock_guard<std::shared_mutex> guard(_mutex); auto it = _documentDBMap.find(docTypeName); if (it == _documentDBMap.end()) { return; @@ -645,7 +645,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) if (_persistenceEngine) { { // Not allowed to get to service layer to call pause(). - std::unique_lock<std::shared_timed_mutex> persistenceWguard(_persistenceEngine->getWLock()); + std::unique_lock<std::shared_mutex> persistenceWguard(_persistenceEngine->getWLock()); IPersistenceHandler::SP oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName); if (_initComplete && oldHandler) { // TODO: Fix race with bucket db modifying ops. @@ -757,7 +757,7 @@ Proton::updateMetrics(const vespalib::MonitorGuard &) void Proton::waitForInitDone() { - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &kv : _documentDBMap) { kv.second->waitForInitDone(); } @@ -766,7 +766,7 @@ Proton::waitForInitDone() void Proton::waitForOnlineState() { - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &kv : _documentDBMap) { kv.second->waitForOnlineState(); } @@ -778,7 +778,7 @@ Proton::getComponentConfig(Consumer &consumer) _protonConfigurer.getComponentConfig().getComponentConfig(consumer); std::vector<DocumentDB::SP> dbs; { - std::shared_lock<std::shared_timed_mutex> guard(_mutex); + std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &kv : _documentDBMap) { dbs.push_back(kv.second); } @@ -847,12 +847,12 @@ struct StateExplorerProxy : vespalib::StateExplorer { struct DocumentDBMapExplorer : vespalib::StateExplorer { typedef std::map<DocTypeName, DocumentDB::SP> DocumentDBMap; const DocumentDBMap &documentDBMap; - std::shared_timed_mutex &mutex; - DocumentDBMapExplorer(const DocumentDBMap &documentDBMap_in, std::shared_timed_mutex &mutex_in) + std::shared_mutex &mutex; + DocumentDBMapExplorer(const DocumentDBMap &documentDBMap_in, std::shared_mutex &mutex_in) : documentDBMap(documentDBMap_in), mutex(mutex_in) {} void get_state(const vespalib::slime::Inserter &, bool) const override {} std::vector<vespalib::string> get_children_names() const override { - std::shared_lock<std::shared_timed_mutex> guard(mutex); + std::shared_lock<std::shared_mutex> guard(mutex); std::vector<vespalib::string> names; for (const auto &item: documentDBMap) { names.push_back(item.first.getName()); @@ -861,7 +861,7 @@ struct DocumentDBMapExplorer : vespalib::StateExplorer { } std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override { typedef std::unique_ptr<StateExplorer> Explorer_UP; - std::shared_lock<std::shared_timed_mutex> guard(mutex); + std::shared_lock<std::shared_mutex> guard(mutex); auto result = documentDBMap.find(DocTypeName(vespalib::string(name))); if (result == documentDBMap.end()) { return Explorer_UP(nullptr); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 55fd3594463..45556319e78 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -86,7 +86,7 @@ private: }; const config::ConfigUri _configUri; - mutable std::shared_timed_mutex _mutex; + mutable std::shared_mutex _mutex; MetricsUpdateHook _metricsHook; std::unique_ptr<MetricsEngine> _metricsEngine; ProtonFileHeaderContext _fileHeaderContext; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp index 31fd44eec5e..23289296ada 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp @@ -2,15 +2,13 @@ #include "proton_disk_layout.h" #include <vespa/vespalib/io/fileutil.h> -#include <vespa/fastos/file.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchlib/transactionlog/translogclient.h> -#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".proton.server.proton_disk_layout"); -using search::transactionlog::TransLogClient; +using search::transactionlog::client::TransLogClient; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 29e39e40dfd..0fcf9b99718 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -3,7 +3,6 @@ #include "executorthreadingservice.h" #include "fast_access_doc_subdb.h" -#include "feedhandler.h" #include "searchable_doc_subdb_configurer.h" #include "searchable_feed_view.h" #include "searchview.h" diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp deleted file mode 100644 index baba74c482c..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tlcproxy.h" -#include <vespa/searchcore/proton/feedoperation/feedoperation.h> - -#include <vespa/log/log.h> -LOG_SETUP(".proton.server.tlcproxy"); - -using vespalib::nbostream; -using search::transactionlog::Packet; - -namespace proton { - -void TlcProxy::commit(search::SerialNum serialNum, search::transactionlog::Type type, - const vespalib::nbostream &buf, DoneCallback onDone) -{ - Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.data(), buf.size())); - Packet packet(entry.serializedSize()); - packet.add(entry); - _tlsDirectWriter.commit(_domain, packet, std::move(onDone)); -} - -void -TlcProxy::storeOperation(const FeedOperation &op, DoneCallback onDone) -{ - nbostream stream; - op.serialize(stream); - LOG(debug, "storeOperation(): serialNum(%" PRIu64 "), type(%u), size(%zu)", - op.getSerialNum(), (uint32_t)op.getType(), stream.size()); - commit(op.getSerialNum(), (uint32_t)op.getType(), stream, std::move(onDone)); -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h deleted file mode 100644 index 2dc6501731e..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/searchlib/transactionlog/common.h> - -namespace proton { - -class FeedOperation; - -class TlcProxy { - using DoneCallback = search::transactionlog::Writer::DoneCallback; - using Writer = search::transactionlog::Writer; - vespalib::string _domain; - Writer & _tlsDirectWriter; - - void commit(search::SerialNum serialNum, search::transactionlog::Type type, - const vespalib::nbostream &buf, DoneCallback onDone); -public: - typedef std::unique_ptr<TlcProxy> UP; - - TlcProxy(const vespalib::string & domain, Writer & writer) - : _domain(domain), _tlsDirectWriter(writer) {} - - void storeOperation(const FeedOperation &op, DoneCallback onDone); -}; - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp index 3ad98cba3ac..fdc9b6d7807 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp @@ -2,6 +2,7 @@ #include "configstore.h" #include "transactionlogmanager.h" +#include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/exceptions.h> @@ -11,11 +12,11 @@ LOG_SETUP(".proton.server.transactionlogmanager"); using vespalib::IllegalStateException; using vespalib::make_string; -using search::transactionlog::TransLogClient; +using search::transactionlog::client::TransLogClient; +using search::transactionlog::client::Session; namespace proton { - void TransactionLogManager::doLogReplayComplete(const vespalib::string &domainName, vespalib::duration elapsedTime) const @@ -45,10 +46,8 @@ TransactionLogManager::init(SerialNum oldestConfigSerial, SerialNum &prunedSeria namespace { -void getStatus(TransLogClient::Session & session, - search::SerialNum & serialBegin, - search::SerialNum & serialEnd, - size_t & count) +void +getStatus(Session & session, search::SerialNum & serialBegin, search::SerialNum & serialEnd, size_t & count) { if (!session.status(serialBegin, serialEnd, count)) { throw IllegalStateException( @@ -66,7 +65,7 @@ void getStatus(TransLogClient & client, search::SerialNum & serialEnd, size_t & count) { - TransLogClient::Session::UP session = client.open(domainName); + std::unique_ptr<Session> session = client.open(domainName); if ( ! session) { throw IllegalStateException( make_string( @@ -117,7 +116,7 @@ TransactionLogManager::prepareReplay(TransLogClient &client, TlsReplayProgress::UP TransactionLogManager::startReplay(SerialNum first, SerialNum syncToken, - TransLogClient::Session::Callback &callback) + Callback &callback) { assert( !_visitor); _visitor = createTlcVisitor(callback); @@ -142,7 +141,7 @@ TransactionLogManager::startReplay(SerialNum first, getDomainName().c_str(), first, syncToken, getRpcTarget().c_str())); } - return TlsReplayProgress::UP(new TlsReplayProgress(getDomainName(), first, syncToken)); + return std::make_unique<TlsReplayProgress>(getDomainName(), first, syncToken); } diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.h b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.h index 15666c38483..32532e3f656 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.h @@ -2,11 +2,8 @@ #pragma once -#include "documentdbconfig.h" #include "tls_replay_progress.h" #include "transactionlogmanagerbase.h" -#include <vespa/searchcore/proton/index/i_index_writer.h> -#include <vespa/searchlib/transactionlog/translogclient.h> namespace proton { struct ConfigStore; @@ -16,7 +13,7 @@ struct ConfigStore; **/ class TransactionLogManager : public TransactionLogManagerBase { - TransLogClient::Visitor::UP _visitor; + std::unique_ptr<Visitor> _visitor; void doLogReplayComplete(const vespalib::string &domainName, vespalib::duration elapsedTime) const override; @@ -54,7 +51,7 @@ public: /** * Start replay of the transaction log. **/ - TlsReplayProgress::UP startReplay(SerialNum first, SerialNum syncToken, TransLogClient::Session::Callback &callback); + TlsReplayProgress::UP startReplay(SerialNum first, SerialNum syncToken, Callback &callback); /** * Indicate that replay is done. diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp index 8b18a7ae566..a8ecb2ba07b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.cpp @@ -1,19 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "transactionlogmanagerbase.h" +#include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.transactionlogmanagerbase"); -using search::transactionlog::TransLogClient; +using search::transactionlog::client::Visitor; namespace proton { TransactionLogManagerBase::TransactionLogManagerBase( const vespalib::string &tlsSpec, const vespalib::string &domainName) : - _tlc(tlsSpec), + _tlc(std::make_unique<TransLogClient>(tlsSpec)), _tlcSession(), _domainName(domainName), _replayLock(), @@ -29,31 +30,31 @@ TransactionLogManagerBase::~TransactionLogManagerBase() = default; TransactionLogManagerBase::StatusResult TransactionLogManagerBase::init() { - TransLogClient::Session::UP session = _tlc.open(_domainName); + std::unique_ptr<Session> session = _tlc->open(_domainName); if ( ! session) { - if (!_tlc.create(_domainName)) { + if (!_tlc->create(_domainName)) { vespalib::string str = vespalib::make_string( "Failed creating domain '%s' on TLS '%s'", - _domainName.c_str(), _tlc.getRPCTarget().c_str()); + _domainName.c_str(), _tlc->getRPCTarget().c_str()); throw std::runtime_error(str); } LOG(debug, "Created domain '%s' on TLS '%s'", - _domainName.c_str(), _tlc.getRPCTarget().c_str()); - session = _tlc.open(_domainName); + _domainName.c_str(), _tlc->getRPCTarget().c_str()); + session = _tlc->open(_domainName); if ( ! session) { vespalib::string str = vespalib::make_string( "Could not open session for domain '%s' on TLS '%s'", - _domainName.c_str(), _tlc.getRPCTarget().c_str()); + _domainName.c_str(), _tlc->getRPCTarget().c_str()); throw std::runtime_error(str); } } LOG(debug, "Opened domain '%s' on TLS '%s'", - _domainName.c_str(), _tlc.getRPCTarget().c_str()); + _domainName.c_str(), _tlc->getRPCTarget().c_str()); StatusResult res; if (!session->status(res.serialBegin, res.serialEnd, res.count)) { vespalib::string str = vespalib::make_string( "Could not get status from session with domain '%s' on TLS '%s'", - _domainName.c_str(), _tlc.getRPCTarget().c_str()); + _domainName.c_str(), _tlc->getRPCTarget().c_str()); throw std::runtime_error(str); } LOG(debug, @@ -72,7 +73,8 @@ TransactionLogManagerBase::internalStartReplay() _replayStopWatch = vespalib::Timer(); } -void TransactionLogManagerBase::changeReplayDone() +void +TransactionLogManagerBase::changeReplayDone() { std::lock_guard<std::mutex> guard(_replayLock); _replayDone = true; @@ -101,23 +103,31 @@ TransactionLogManagerBase::close() } } -TransLogClient::Visitor::UP -TransactionLogManagerBase::createTlcVisitor(TransLogClient::Session::Callback &callback) { - return _tlc.createVisitor(_domainName, callback); +std::unique_ptr<Visitor> +TransactionLogManagerBase::createTlcVisitor(Callback &callback) { + return _tlc->createVisitor(_domainName, callback); } -bool TransactionLogManagerBase::getReplayDone() const { +bool +TransactionLogManagerBase::getReplayDone() const { std::lock_guard<std::mutex> guard(_replayLock); return _replayDone; } -bool TransactionLogManagerBase::isDoingReplay() const { +bool +TransactionLogManagerBase::isDoingReplay() const { std::lock_guard<std::mutex> guard(_replayLock); return _replayStarted && !_replayDone; } -void TransactionLogManagerBase::logReplayComplete() const { +void +TransactionLogManagerBase::logReplayComplete() const { doLogReplayComplete(_domainName, _replayStopWatch.elapsed()); } +const vespalib::string & +TransactionLogManagerBase::getRpcTarget() const { + return _tlc->getRPCTarget(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h index 4b5d001a28e..7059604dfe7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanagerbase.h @@ -2,11 +2,17 @@ #pragma once -#include <vespa/searchlib/transactionlog/translogclient.h> +#include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/util/time.h> #include <mutex> #include <condition_variable> +namespace search::transactionlog::client { + class TransLogClient; + class Session; + class Visitor; + class Callback; +} namespace proton { /** @@ -14,10 +20,13 @@ namespace proton { **/ class TransactionLogManagerBase { protected: - using TransLogClient = search::transactionlog::TransLogClient; + using TransLogClient = search::transactionlog::client::TransLogClient; + using Session = search::transactionlog::client::Session; + using Visitor = search::transactionlog::client::Visitor; + using Callback = search::transactionlog::client::Callback; private: - TransLogClient _tlc; - TransLogClient::Session::UP _tlcSession; + std::unique_ptr<TransLogClient> _tlc; + std::unique_ptr<Session> _tlcSession; vespalib::string _domainName; mutable std::mutex _replayLock; mutable std::condition_variable _replayCond; @@ -26,7 +35,7 @@ private: vespalib::Timer _replayStopWatch; protected: - typedef search::SerialNum SerialNum; + using SerialNum = search::SerialNum; struct StatusResult { SerialNum serialBegin; @@ -55,17 +64,17 @@ public: void changeReplayDone(); void close(); - TransLogClient::Visitor::UP createTlcVisitor(TransLogClient::Session::Callback &callback); + std::unique_ptr<Visitor> createTlcVisitor(Callback &callback); void waitForReplayDone() const; - TransLogClient &getClient() { return _tlc; } - TransLogClient::Session *getSession() { return _tlcSession.get(); } + TransLogClient &getClient() { return *_tlc; } + Session *getSession() { return _tlcSession.get(); } const vespalib::string &getDomainName() const { return _domainName; } bool getReplayDone() const; bool isDoingReplay() const; void logReplayComplete() const; - const vespalib::string &getRpcTarget() const { return _tlc.getRPCTarget(); } + const vespalib::string &getRpcTarget() const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.h b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.h index 560b6e75423..c22128685c1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.h @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/common/icommitable.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/vespalib/util/varholder.h> +#include <vespa/vespalib/util/time.h> #include <mutex> namespace proton { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 6a9108311b1..f1c421f6c22 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; */ public class EvaluationTestCase { - private double tolerance = 0.000001; + private final double tolerance = 0.000001; @Test public void testEvaluation() { diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index a20e0cc3aaa..80142f705c1 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/document/util/bytebuffer.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> @@ -15,17 +16,22 @@ using namespace document; using namespace vespalib; using namespace std::chrono_literals; using search::index::DummyFileHeaderContext; +using search::transactionlog::client::TransLogClient; +using search::transactionlog::client::Session; +using search::transactionlog::client::Visitor; +using search::transactionlog::client::RPC; +using search::transactionlog::client::Callback; namespace { bool createDomainTest(TransLogClient & tls, const vespalib::string & name, size_t preExistingDomains=0); -TransLogClient::Session::UP openDomainTest(TransLogClient & tls, const vespalib::string & name); -bool fillDomainTest(TransLogClient::Session * s1, const vespalib::string & name); -void fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_t numEntries); -void fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_t numEntries, size_t entrySize); +std::unique_ptr<Session> openDomainTest(TransLogClient & tls, const vespalib::string & name); +bool fillDomainTest(Session * s1, const vespalib::string & name); +void fillDomainTest(Session * s1, size_t numPackets, size_t numEntries); +void fillDomainTest(Session * s1, size_t numPackets, size_t numEntries, size_t entrySize); uint32_t countFiles(const vespalib::string &dir); -void checkFilledDomainTest(const TransLogClient::Session::UP &s1, size_t numEntries); -bool visitDomainTest(TransLogClient & tls, TransLogClient::Session * s1, const vespalib::string & name); +void checkFilledDomainTest(Session &s1, size_t numEntries); +bool visitDomainTest(TransLogClient & tls, Session * s1, const vespalib::string & name); void createAndFillDomain(const vespalib::string & name, Encoding encoding, size_t preExistingDomains); void verifyDomain(const vespalib::string & name); @@ -33,7 +39,7 @@ vespalib::string myhex(const void * b, size_t sz) { static const char * hextab="0123456789ABCDEF"; - const unsigned char * c = static_cast<const unsigned char *>(b); + const auto * c = static_cast<const unsigned char *>(b); vespalib::string s; s.reserve(sz*2); for (size_t i=0; i < sz; i++) { @@ -43,11 +49,11 @@ myhex(const void * b, size_t sz) return s; } -class CallBackTest : public TransLogClient::Visitor::Callback +class CallBackTest : public Callback { private: - virtual RPC::Result receive(const Packet & packet) override; - virtual void eof() override { _eof = true; } + RPC::Result receive(const Packet & packet) override; + void eof() override { _eof = true; } typedef std::map<SerialNum, std::unique_ptr<ByteBuffer>> PacketMap; PacketMap _packetMap; public: @@ -65,7 +71,7 @@ CallBackTest::receive(const Packet & p) { nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(h.peek(), h.size()).c_str()); - while(h.size() > 0) { + while( ! h.empty()) { Packet::Entry e; e.deserialize(h); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(e.data().c_str(), e.data().size()).c_str()); @@ -74,13 +80,13 @@ CallBackTest::receive(const Packet & p) return RPC::OK; } -class CallBackManyTest : public TransLogClient::Visitor::Callback +class CallBackManyTest : public Callback { private: - virtual RPC::Result receive(const Packet & packet) override; - virtual void eof() override { _eof = true; } + RPC::Result receive(const Packet & packet) override; + void eof() override { _eof = true; } public: - CallBackManyTest(size_t start) : _eof(false), _count(start), _value(start) { } + explicit CallBackManyTest(size_t start) : _eof(false), _count(start), _value(start) { } void clear() { _eof = false; _count = 0; _value = 0; } bool _eof; size_t _count; @@ -91,7 +97,7 @@ RPC::Result CallBackManyTest::receive(const Packet & p) { nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); - for(;h.size() > 0; _count++, _value++) { + for(; ! h.empty(); _count++, _value++) { Packet::Entry e; e.deserialize(h); assert(e.data().size() == 8); @@ -103,17 +109,22 @@ CallBackManyTest::receive(const Packet & p) return RPC::OK; } -class CallBackUpdate : public TransLogClient::Visitor::Callback +class CallBackUpdate : public Callback { public: typedef std::map<SerialNum, Identifiable *> PacketMap; private: - virtual RPC::Result receive(const Packet & packet) override; - virtual void eof() override { _eof = true; } + RPC::Result receive(const Packet & packet) override; + void eof() override { _eof = true; } PacketMap _packetMap; public: CallBackUpdate() : _eof(false) { } - virtual ~CallBackUpdate() { while (_packetMap.begin() != _packetMap.end()) { delete _packetMap.begin()->second; _packetMap.erase(_packetMap.begin()); } } + ~CallBackUpdate() override { + while (_packetMap.begin() != _packetMap.end()) { + delete _packetMap.begin()->second; + _packetMap.erase(_packetMap.begin()); + } + } bool hasSerial(SerialNum n) const { return (_packetMap.find(n) != _packetMap.end()); } const PacketMap & map() const { return _packetMap; } bool _eof; @@ -124,14 +135,14 @@ RPC::Result CallBackUpdate::receive(const Packet & packet) { nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); - while (h.size() > 0) { + while ( ! h.empty() ) { Packet::Entry e; e.deserialize(h); const vespalib::Identifiable::RuntimeClass * cl(vespalib::Identifiable::classFromId(e.type())); if (cl) { vespalib::Identifiable * obj(cl->create()); if (obj->inherits(Identifiable::classId)) { - Identifiable * ser = static_cast<Identifiable *>(obj); + auto * ser = static_cast<Identifiable *>(obj); nbostream is(e.data().c_str(), e.data().size()); try { is >> *ser; @@ -140,8 +151,8 @@ CallBackUpdate::receive(const Packet & packet) assert(false); return RPC::ERROR; } - assert(is.state() == nbostream::ok); - assert(is.size() == 0); + ASSERT_TRUE(is.state() == nbostream::ok); + ASSERT_TRUE(is.empty()); _packetMap[e.serial()] = ser; } else { LOG(warning, "Packet::Entry(%" PRId64 ", %s) is not a Identifiable", e.serial(), cl->name()); @@ -153,11 +164,11 @@ CallBackUpdate::receive(const Packet & packet) return RPC::OK; } -class CallBackStatsTest : public TransLogClient::Session::Callback +class CallBackStatsTest : public Callback { private: - virtual RPC::Result receive(const Packet & packet) override; - virtual void eof() override { _eof = true; } + RPC::Result receive(const Packet & packet) override; + void eof() override { _eof = true; } public: CallBackStatsTest() : _eof(false), _count(0), _inOrder(0), @@ -177,7 +188,7 @@ RPC::Result CallBackStatsTest::receive(const Packet & p) { nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); - for(;h.size() > 0; ++_count) { + for(; ! h.empty(); ++_count) { Packet::Entry e; e.deserialize(h); SerialNum s = e.serial(); @@ -219,8 +230,8 @@ createDomainTest(TransLogClient & tls, const vespalib::string & name, size_t pre std::vector<vespalib::string> dir; tls.listDomains(dir); EXPECT_EQUAL (dir.size(), preExistingDomains); - TransLogClient::Session::UP s1 = tls.open(name); - ASSERT_TRUE (s1.get() == NULL); + auto s1 = tls.open(name); + ASSERT_FALSE (s1); retval = tls.create(name); ASSERT_TRUE (retval); dir.clear(); @@ -230,16 +241,16 @@ createDomainTest(TransLogClient & tls, const vespalib::string & name, size_t pre return retval; } -TransLogClient::Session::UP +std::unique_ptr<Session> openDomainTest(TransLogClient & tls, const vespalib::string & name) { - TransLogClient::Session::UP s1 = tls.open(name); - ASSERT_TRUE (s1.get() != NULL); + auto s1 = tls.open(name); + ASSERT_TRUE (s1); return s1; } bool -fillDomainTest(TransLogClient::Session * s1, const vespalib::string & name) +fillDomainTest(Session * s1, const vespalib::string & name) { bool retval(true); Packet::Entry e1(1, 1, vespalib::ConstBufferRef("Content in buffer A", 20)); @@ -279,7 +290,7 @@ fillDomainTest(TransLogClient::Session * s1, const vespalib::string & name) } void -fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_t numEntries) +fillDomainTest(Session * s1, size_t numPackets, size_t numEntries) { size_t value(0); for(size_t i=0; i < numPackets; i++) { @@ -289,7 +300,7 @@ fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_t numEntrie p->add(e); if (p->sizeBytes() > DEFAULT_PACKET_SIZE){ ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); - p.reset(new Packet(DEFAULT_PACKET_SIZE)); + p = std::make_unique<Packet>(DEFAULT_PACKET_SIZE); } } ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); @@ -300,7 +311,7 @@ using Counter = std::atomic<size_t>; class CountDone : public IDestructorCallback { public: - CountDone(Counter & inFlight) : _inFlight(inFlight) { ++_inFlight; } + explicit CountDone(Counter & inFlight) : _inFlight(inFlight) { ++_inFlight; } ~CountDone() override { --_inFlight; } private: Counter & _inFlight; @@ -311,17 +322,18 @@ fillDomainTest(TransLogServer & s1, const vespalib::string & domain, size_t numP { size_t value(0); Counter inFlight(0); + auto domainWriter = s1.getWriter(domain); for(size_t i=0; i < numPackets; i++) { std::unique_ptr<Packet> p(new Packet(DEFAULT_PACKET_SIZE)); for(size_t j=0; j < numEntries; j++, value++) { Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&value, sizeof(value))); p->add(e); if ( p->sizeBytes() > DEFAULT_PACKET_SIZE ) { - s1.commit(domain, *p, std::make_shared<CountDone>(inFlight)); - p.reset(new Packet(DEFAULT_PACKET_SIZE)); + domainWriter->commit(*p, std::make_shared<CountDone>(inFlight)); + p = std::make_unique<Packet>(DEFAULT_PACKET_SIZE); } } - s1.commit(domain, *p, std::make_shared<CountDone>(inFlight)); + domainWriter->commit(*p, std::make_shared<CountDone>(inFlight)); LOG(info, "Inflight %ld", inFlight.load()); } while (inFlight.load() != 0) { @@ -333,7 +345,7 @@ fillDomainTest(TransLogServer & s1, const vespalib::string & domain, size_t numP void -fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_t numEntries, size_t entrySize) +fillDomainTest(Session * s1, size_t numPackets, size_t numEntries, size_t entrySize) { size_t value(0); std::vector<char> entryBuffer(entrySize); @@ -344,7 +356,7 @@ fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_t numEntrie p->add(e); if (p->sizeBytes() > DEFAULT_PACKET_SIZE){ ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); - p.reset(new Packet(DEFAULT_PACKET_SIZE)); + p = std::make_unique<Packet>(DEFAULT_PACKET_SIZE); } } ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); @@ -368,18 +380,18 @@ countFiles(const vespalib::string &dir) } void -checkFilledDomainTest(const TransLogClient::Session::UP &s1, size_t numEntries) +checkFilledDomainTest(Session &s1, size_t numEntries) { SerialNum b(0), e(0); size_t c(0); - EXPECT_TRUE(s1->status(b, e, c)); + EXPECT_TRUE(s1.status(b, e, c)); EXPECT_EQUAL(b, 1u); EXPECT_EQUAL(e, numEntries); EXPECT_EQUAL(c, numEntries); } bool -visitDomainTest(TransLogClient & tls, TransLogClient::Session * s1, const vespalib::string & name) +visitDomainTest(TransLogClient & tls, Session * s1, const vespalib::string & name) { bool retval(true); @@ -391,8 +403,8 @@ visitDomainTest(TransLogClient & tls, TransLogClient::Session * s1, const vespal EXPECT_EQUAL(c, 3u); CallBackTest ca; - TransLogClient::Visitor::UP visitor = tls.createVisitor(name, ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor(name, ca); + ASSERT_TRUE(visitor); EXPECT_TRUE( visitor->visit(0, 1) ); for (size_t i(0); ! ca._eof && (i < 60000); i++ ) { std::this_thread::sleep_for(10ms); } EXPECT_TRUE( ca._eof ); @@ -451,7 +463,7 @@ void createAndFillDomain(const vespalib::string & name, Encoding encoding, size_ TransLogClient tls("tcp/localhost:18377"); createDomainTest(tls, name, preExistingDomains); - TransLogClient::Session::UP s1 = openDomainTest(tls, name); + auto s1 = openDomainTest(tls, name); fillDomainTest(s1.get(), name); } @@ -459,7 +471,7 @@ void verifyDomain(const vespalib::string & name) { DummyFileHeaderContext fileHeaderContext; TransLogServer tlss("test13", 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x1000000)); TransLogClient tls("tcp/localhost:18377"); - TransLogClient::Session::UP s1 = openDomainTest(tls, name); + auto s1 = openDomainTest(tls, name); visitDomainTest(tls, s1.get(), name); } @@ -472,7 +484,7 @@ TEST("testVisitOverGeneratedDomain") { vespalib::string name("test1"); createDomainTest(tls, name); - TransLogClient::Session::UP s1 = openDomainTest(tls, name); + auto s1 = openDomainTest(tls, name); fillDomainTest(s1.get(), name); EXPECT_EQUAL(0, getMaxSessionRunTime(tlss, "test1")); visitDomainTest(tls, s1.get(), name); @@ -488,7 +500,7 @@ TEST("testVisitOverPreExistingDomain") { TransLogClient tls("tcp/localhost:18377"); vespalib::string name("test1"); - TransLogClient::Session::UP s1 = openDomainTest(tls, name); + auto s1 = openDomainTest(tls, name); visitDomainTest(tls, s1.get(), name); } @@ -497,8 +509,8 @@ TEST("partialUpdateTest") { TransLogServer tlss("test7", 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x10000)); TransLogClient tls("tcp/localhost:18377"); - TransLogClient::Session::UP s1 = openDomainTest(tls, "test1"); - TransLogClient::Session & session = *s1; + auto s1 = openDomainTest(tls, "test1"); + Session & session = *s1; TestIdentifiable du; @@ -513,8 +525,8 @@ TEST("partialUpdateTest") { ASSERT_TRUE(session.commit(vespalib::ConstBufferRef(pa.getHandle().data(), pa.getHandle().size()))); CallBackUpdate ca; - TransLogClient::Visitor::UP visitor = tls.createVisitor("test1", ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor("test1", ca); + ASSERT_TRUE(visitor); ASSERT_TRUE( visitor->visit(5, 7) ); for (size_t i(0); ! ca._eof && (i < 1000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca._eof ); @@ -522,24 +534,24 @@ TEST("partialUpdateTest") { ASSERT_TRUE( ca.hasSerial(7) ); CallBackUpdate ca1; - TransLogClient::Visitor::UP visitor1 = tls.createVisitor("test1", ca1); - ASSERT_TRUE(visitor1.get()); + auto visitor1 = tls.createVisitor("test1", ca1); + ASSERT_TRUE(visitor1); ASSERT_TRUE( visitor1->visit(4, 5) ); for (size_t i(0); ! ca1._eof && (i < 1000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca1._eof ); - ASSERT_TRUE( ca1.map().size() == 0); + ASSERT_TRUE( ca1.map().empty()); CallBackUpdate ca2; - TransLogClient::Visitor::UP visitor2 = tls.createVisitor("test1", ca2); - ASSERT_TRUE(visitor2.get()); + auto visitor2 = tls.createVisitor("test1", ca2); + ASSERT_TRUE(visitor2); ASSERT_TRUE( visitor2->visit(5, 6) ); for (size_t i(0); ! ca2._eof && (i < 1000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca2._eof ); - ASSERT_TRUE( ca2.map().size() == 0); + ASSERT_TRUE( ca2.map().empty()); CallBackUpdate ca3; - TransLogClient::Visitor::UP visitor3 = tls.createVisitor("test1", ca3); - ASSERT_TRUE(visitor3.get()); + auto visitor3 = tls.createVisitor("test1", ca3); + ASSERT_TRUE(visitor3); ASSERT_TRUE( visitor3->visit(5, 1000) ); for (size_t i(0); ! ca3._eof && (i < 1000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca3._eof ); @@ -562,7 +574,7 @@ TEST("testRemove") { vespalib::string name("test-delete"); createDomainTest(tls, name); - TransLogClient::Session::UP s1 = openDomainTest(tls, name); + auto s1 = openDomainTest(tls, name); fillDomainTest(s1.get(), name); visitDomainTest(tls, s1.get(), name); ASSERT_TRUE(tls.remove(name)); @@ -577,8 +589,8 @@ assertVisitStats(TransLogClient &tls, const vespalib::string &domain, uint64_t expCount, uint64_t expInOrder) { CallBackStatsTest ca; - TransLogClient::Visitor::UP visitor = tls.createVisitor(domain, ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor(domain, ca); + ASSERT_TRUE(visitor); ASSERT_TRUE( visitor->visit(visitStart, visitEnd) ); for (size_t i(0); ! ca._eof && (i < 60000); i++ ) { std::this_thread::sleep_for(10ms); @@ -591,9 +603,7 @@ assertVisitStats(TransLogClient &tls, const vespalib::string &domain, } void -assertStatus(TransLogClient::Session &s, - SerialNum expFirstSerial, SerialNum expLastSerial, - uint64_t expCount) +assertStatus(Session &s, SerialNum expFirstSerial, SerialNum expLastSerial, uint64_t expCount) { SerialNum b(0), e(0); size_t c(0); @@ -618,7 +628,7 @@ TEST("test sending a lot of data") { TransLogClient tls("tcp/localhost:18377"); createDomainTest(tls, MANY, 0); - TransLogClient::Session::UP s1 = openDomainTest(tls, MANY); + auto s1 = openDomainTest(tls, MANY); fillDomainTest(s1.get(), NUM_PACKETS, NUM_ENTRIES); SerialNum b(0), e(0); size_t c(0); @@ -627,8 +637,8 @@ TEST("test sending a lot of data") { EXPECT_EQUAL(e, TOTAL_NUM_ENTRIES); EXPECT_EQUAL(c, TOTAL_NUM_ENTRIES); CallBackManyTest ca(2); - TransLogClient::Visitor::UP visitor = tls.createVisitor("many", ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor("many", ca); + ASSERT_TRUE(visitor); ASSERT_TRUE( visitor->visit(2, TOTAL_NUM_ENTRIES) ); for (size_t i(0); ! ca._eof && (i < 60000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca._eof ); @@ -640,7 +650,7 @@ TEST("test sending a lot of data") { TransLogServer tlss("test8", 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x1000000)); TransLogClient tls("tcp/localhost:18377"); - TransLogClient::Session::UP s1 = openDomainTest(tls, "many"); + auto s1 = openDomainTest(tls, "many"); SerialNum b(0), e(0); size_t c(0); EXPECT_TRUE(s1->status(b, e, c)); @@ -648,8 +658,8 @@ TEST("test sending a lot of data") { EXPECT_EQUAL(e, TOTAL_NUM_ENTRIES); EXPECT_EQUAL(c, TOTAL_NUM_ENTRIES); CallBackManyTest ca(2); - TransLogClient::Visitor::UP visitor = tls.createVisitor(MANY, ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor(MANY, ca); + ASSERT_TRUE(visitor); ASSERT_TRUE( visitor->visit(2, TOTAL_NUM_ENTRIES) ); for (size_t i(0); ! ca._eof && (i < 60000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca._eof ); @@ -661,7 +671,7 @@ TEST("test sending a lot of data") { TransLogServer tlss("test8", 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x1000000)); TransLogClient tls("tcp/localhost:18377"); - TransLogClient::Session::UP s1 = openDomainTest(tls, MANY); + auto s1 = openDomainTest(tls, MANY); SerialNum b(0), e(0); size_t c(0); EXPECT_TRUE(s1->status(b, e, c)); @@ -669,8 +679,8 @@ TEST("test sending a lot of data") { EXPECT_EQUAL(e, TOTAL_NUM_ENTRIES); EXPECT_EQUAL(c, TOTAL_NUM_ENTRIES); CallBackManyTest ca(2); - TransLogClient::Visitor::UP visitor = tls.createVisitor(MANY, ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor(MANY, ca); + ASSERT_TRUE(visitor); ASSERT_TRUE( visitor->visit(2, TOTAL_NUM_ENTRIES) ); for (size_t i(0); ! ca._eof && (i < 60000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca._eof ); @@ -690,7 +700,7 @@ TEST("test sending a lot of data async") { .setChunkAgeLimit(10ms)); TransLogClient tls("tcp/localhost:18377"); createDomainTest(tls, MANY, 1); - TransLogClient::Session::UP s1 = openDomainTest(tls, MANY); + auto s1 = openDomainTest(tls, MANY); fillDomainTest(tlss, MANY, NUM_PACKETS, NUM_ENTRIES); SerialNum b(0), e(0); size_t c(0); @@ -699,8 +709,8 @@ TEST("test sending a lot of data async") { EXPECT_EQUAL(e, TOTAL_NUM_ENTRIES); EXPECT_EQUAL(c, TOTAL_NUM_ENTRIES); CallBackManyTest ca(2); - TransLogClient::Visitor::UP visitor = tls.createVisitor(MANY, ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor(MANY, ca); + ASSERT_TRUE(visitor); ASSERT_TRUE( visitor->visit(2, TOTAL_NUM_ENTRIES) ); for (size_t i(0); ! ca._eof && (i < 60000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca._eof ); @@ -712,7 +722,7 @@ TEST("test sending a lot of data async") { TransLogServer tlss("test8", 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x1000000)); TransLogClient tls("tcp/localhost:18377"); - TransLogClient::Session::UP s1 = openDomainTest(tls, MANY); + auto s1 = openDomainTest(tls, MANY); SerialNum b(0), e(0); size_t c(0); EXPECT_TRUE(s1->status(b, e, c)); @@ -720,8 +730,8 @@ TEST("test sending a lot of data async") { EXPECT_EQUAL(e, TOTAL_NUM_ENTRIES); EXPECT_EQUAL(c, TOTAL_NUM_ENTRIES); CallBackManyTest ca(2); - TransLogClient::Visitor::UP visitor = tls.createVisitor(MANY, ca); - ASSERT_TRUE(visitor.get()); + auto visitor = tls.createVisitor(MANY, ca); + ASSERT_TRUE(visitor); ASSERT_TRUE( visitor->visit(2, TOTAL_NUM_ENTRIES) ); for (size_t i(0); ! ca._eof && (i < 60000); i++ ) { std::this_thread::sleep_for(10ms); } ASSERT_TRUE( ca._eof ); @@ -743,7 +753,7 @@ TEST("testErase") { TransLogClient tls("tcp/localhost:18377"); createDomainTest(tls, "erase", 0); - TransLogClient::Session::UP s1 = openDomainTest(tls, "erase"); + auto s1 = openDomainTest(tls, "erase"); fillDomainTest(s1.get(), NUM_PACKETS, NUM_ENTRIES); } { @@ -751,7 +761,7 @@ TEST("testErase") { TransLogServer tlss("test12", 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x1000000)); TransLogClient tls("tcp/localhost:18377"); - TransLogClient::Session::UP s1 = openDomainTest(tls, "erase"); + auto s1 = openDomainTest(tls, "erase"); // Before erase TEST_DO(assertVisitStats(tls, "erase", 2, TOTAL_NUM_ENTRIES, @@ -839,7 +849,7 @@ TEST("testSync") { TransLogClient tls("tcp/localhost:18377"); createDomainTest(tls, "sync", 0); - TransLogClient::Session::UP s1 = openDomainTest(tls, "sync"); + auto s1 = openDomainTest(tls, "sync"); fillDomainTest(s1.get(), NUM_PACKETS, NUM_ENTRIES); SerialNum syncedTo(0); @@ -861,7 +871,7 @@ TEST("test truncate on version mismatch") { TransLogClient tls("tcp/localhost:18377"); createDomainTest(tls, "sync", 0); - TransLogClient::Session::UP s1 = openDomainTest(tls, "sync"); + auto s1 = openDomainTest(tls, "sync"); fillDomainTest(s1.get(), NUM_PACKETS, NUM_ENTRIES); EXPECT_TRUE(s1->status(fromOld, toOld, countOld)); SerialNum syncedTo(0); @@ -880,7 +890,7 @@ TEST("test truncate on version mismatch") { { TransLogServer tlss("test11", 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x10000)); TransLogClient tls("tcp/localhost:18377"); - TransLogClient::Session::UP s1 = openDomainTest(tls, "sync"); + auto s1 = openDomainTest(tls, "sync"); uint64_t from(0), to(0); size_t count(0); EXPECT_TRUE(s1->status(from, to, count)); @@ -906,7 +916,7 @@ TEST("test truncation after short read") { TransLogClient tls(tlsspec); createDomainTest(tls, domain, 0); - TransLogClient::Session::UP s1 = openDomainTest(tls, domain); + auto s1 = openDomainTest(tls, domain); fillDomainTest(s1.get(), NUM_PACKETS, NUM_ENTRIES, ENTRYSIZE); SerialNum syncedTo(0); @@ -920,8 +930,8 @@ TEST("test truncation after short read") { { TransLogServer tlss(topdir, 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x10000)); TransLogClient tls(tlsspec); - TransLogClient::Session::UP s1 = openDomainTest(tls, domain); - checkFilledDomainTest(s1, TOTAL_NUM_ENTRIES); + auto s1 = openDomainTest(tls, domain); + checkFilledDomainTest(*s1, TOTAL_NUM_ENTRIES); } { EXPECT_EQUAL(2u, countFiles(dir)); @@ -929,15 +939,15 @@ TEST("test truncation after short read") { { vespalib::string filename(dir + "/truncate-0000000000000017"); FastOS_File trfile(filename.c_str()); - EXPECT_TRUE(trfile.OpenReadWrite(NULL)); + EXPECT_TRUE(trfile.OpenReadWrite(nullptr)); trfile.SetSize(trfile.getSize() - 1); trfile.Close(); } { TransLogServer tlss(topdir, 18377, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(0x10000)); TransLogClient tls(tlsspec); - TransLogClient::Session::UP s1 = openDomainTest(tls, domain); - checkFilledDomainTest(s1, TOTAL_NUM_ENTRIES - 1); + auto s1 = openDomainTest(tls, domain); + checkFilledDomainTest(*s1, TOTAL_NUM_ENTRIES - 1); } { EXPECT_EQUAL(2u, countFiles(dir)); diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 925f297bf48..a516fb26604 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -27,8 +27,10 @@ using search::index::DummyFileHeaderContext; namespace search::transactionlog { -using ClientSession = TransLogClient::Session; -using Visitor = TransLogClient::Visitor; +using ClientSession = client::Session; +using client::Visitor; +using client::TransLogClient; +using client::RPC; //----------------------------------------------------------------------------- // BufferGenerator @@ -287,7 +289,7 @@ FeederThread::doRun() //----------------------------------------------------------------------------- // Agent //----------------------------------------------------------------------------- -class Agent : public ClientSession::Callback +class Agent : public client::Callback { protected: std::string _tlsSpec; @@ -301,12 +303,13 @@ protected: public: Agent(const std::string & tlsSpec, const std::string & domain, const EntryGenerator & generator, const std::string & name, uint32_t id, bool validate) : - ClientSession::Callback(), + client::Callback(), _tlsSpec(tlsSpec), _domain(domain), _client(tlsSpec), - _generator(generator), _name(name), _id(id), _validate(validate) {} - virtual ~Agent() {} - virtual RPC::Result receive(const Packet & packet) override = 0; - virtual void eof() override {} + _generator(generator), _name(name), _id(id), _validate(validate) + {} + ~Agent() override {} + RPC::Result receive(const Packet & packet) override = 0; + void eof() override {} virtual void failed() {} }; diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 08c7186e8c7..dbd5690093b 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -635,7 +635,7 @@ AttributeVector::onInitSave(vespalib::stringref) bool AttributeVector::hasActiveEnumGuards() { - std::unique_lock<std::shared_timed_mutex> lock(_enumLock, std::defer_lock); + std::unique_lock<std::shared_mutex> lock(_enumLock, std::defer_lock); for (size_t i = 0; i < 1000; ++i) { // Note: Need to run this in loop as try_lock() is allowed to fail spuriously and return false // even if the mutex is not currently locked by any other thread. @@ -735,10 +735,10 @@ class ReadGuard : public attribute::AttributeReadGuard { using GenerationHandler = vespalib::GenerationHandler; GenerationHandler::Guard _generationGuard; - using EnumGuard = std::shared_lock<std::shared_timed_mutex>; + using EnumGuard = std::shared_lock<std::shared_mutex>; EnumGuard _enumGuard; public: - ReadGuard(const attribute::IAttributeVector *attr, GenerationHandler::Guard &&generationGuard, std::shared_timed_mutex *enumLock) + ReadGuard(const attribute::IAttributeVector *attr, GenerationHandler::Guard &&generationGuard, std::shared_mutex *enumLock) : attribute::AttributeReadGuard(attr), _generationGuard(std::move(generationGuard)), _enumGuard(enumLock != nullptr ? EnumGuard(*enumLock) : EnumGuard()) diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 75699868691..d8bdda911fd 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -232,9 +232,9 @@ protected: public: class EnumModifier { - std::unique_lock<std::shared_timed_mutex> _enumLock; + std::unique_lock<std::shared_mutex> _enumLock; public: - EnumModifier(std::shared_timed_mutex &lock, attribute::InterlockGuard &interlockGuard) + EnumModifier(std::shared_mutex &lock, attribute::InterlockGuard &interlockGuard) : _enumLock(lock) { (void) interlockGuard; @@ -575,7 +575,7 @@ private: BaseName _baseFileName; Config _config; std::shared_ptr<attribute::Interlock> _interlock; - mutable std::shared_timed_mutex _enumLock; + mutable std::shared_mutex _enumLock; GenerationHandler _genHandler; GenerationHolder _genHolder; Status _status; @@ -606,7 +606,7 @@ private: * Used to regulate access to critical resources. Apply the * reader/writer guards. */ - std::shared_timed_mutex & getEnumLock() { return _enumLock; } + std::shared_mutex & getEnumLock() { return _enumLock; } friend class ComponentGuard<AttributeVector>; friend class AttributeValueGuard; diff --git a/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt b/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt index 5dca84a26c1..6ce7e652326 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(searchlib_transactionlog OBJECT SOURCES chunks.cpp + client_session.cpp common.cpp domain.cpp domainconfig.cpp diff --git a/searchlib/src/vespa/searchlib/transactionlog/client_common.h b/searchlib/src/vespa/searchlib/transactionlog/client_common.h new file mode 100644 index 00000000000..05bb30ff368 --- /dev/null +++ b/searchlib/src/vespa/searchlib/transactionlog/client_common.h @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace search::transactionlog { class Packet; } +namespace search::transactionlog::client { + +class RPC +{ +public: +enum Result { OK, FULL, ERROR }; +}; + +class Callback { +public: + virtual ~Callback() = default; + virtual RPC::Result receive(const Packet & packet) = 0; + virtual void eof() { } +}; + +} diff --git a/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp b/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp new file mode 100644 index 00000000000..8678d88b43c --- /dev/null +++ b/searchlib/src/vespa/searchlib/transactionlog/client_session.cpp @@ -0,0 +1,200 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "client_session.h" +#include "translogclient.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/fnet/frt/supervisor.h> +#include <thread> + +#include <vespa/log/log.h> +LOG_SETUP(".translog.client_session"); + +using vespalib::LockGuard; +using namespace std::chrono_literals; + +namespace search::transactionlog::client { + +SessionKey::SessionKey(const vespalib::string & domain, int sessionId) + : _domain(domain), + _sessionId(sessionId) +{ } +SessionKey::~SessionKey() = default; + +int +SessionKey::cmp(const SessionKey & b) const +{ + int diff(strcmp(_domain.c_str(), b._domain.c_str())); + if (diff == 0) { + diff = _sessionId - b._sessionId; + } + return diff; +} + +Session::Session(const vespalib::string & domain, TransLogClient & tlc) + : _tlc(tlc), + _domain(domain), + _sessionId(0) +{ +} + +Session::~Session() +{ + close(); + clear(); +} + +bool +Session::commit(const vespalib::ConstBufferRef & buf) +{ + bool retval(true); + if (buf.size() != 0) { + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); + req->SetMethodName("domainCommit"); + req->GetParams()->AddString(_domain.c_str()); + req->GetParams()->AddData(buf.c_str(), buf.size()); + int retcode = _tlc.rpc(req); + retval = (retcode == 0); + if (retval) { + req->SubRef(); + } else { + vespalib::string msg; + if (req->GetReturn() != nullptr) { + msg = req->GetReturn()->GetValue(1)._string._str; + } else { + msg = vespalib::make_string("Clientside error %s: error(%d): %s", req->GetMethodName(), req->GetErrorCode(), req->GetErrorMessage()); + } + req->SubRef(); + throw std::runtime_error(vespalib::make_string("commit failed with code %d. server says: %s", retcode, msg.c_str())); + } + } + return retval; +} + +bool +Session::status(SerialNum & b, SerialNum & e, size_t & count) +{ + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); + req->SetMethodName("domainStatus"); + req->GetParams()->AddString(_domain.c_str()); + int32_t retval(_tlc.rpc(req)); + if (retval == 0) { + b = req->GetReturn()->GetValue(1)._intval64; + e = req->GetReturn()->GetValue(2)._intval64; + count = req->GetReturn()->GetValue(3)._intval64; + } + req->SubRef(); + return (retval == 0); +} + +bool +Session::erase(const SerialNum & to) +{ + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); + req->SetMethodName("domainPrune"); + req->GetParams()->AddString(_domain.c_str()); + req->GetParams()->AddInt64(to); + int32_t retval(_tlc.rpc(req)); + req->SubRef(); + if (retval == 1) { + LOG(warning, "Prune to %" PRIu64 " denied since there were active visitors in that area", to); + } + return (retval == 0); +} + + +bool +Session::sync(const SerialNum &syncTo, SerialNum &syncedTo) +{ + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); + req->SetMethodName("domainSync"); + FRT_Values & params = *req->GetParams(); + params.AddString(_domain.c_str()); + params.AddInt64(syncTo); + int32_t retval(_tlc.rpc(req)); + if (retval == 0) { + syncedTo = req->GetReturn()->GetValue(1)._intval64; + } + req->SubRef(); + return (retval == 0); +} + + +void +Session::clear() +{ + if (_sessionId > 0) { + LockGuard guard(_tlc._lock); + _tlc._sessions.erase(SessionKey(_domain, _sessionId)); + } + _sessionId = 0; +} + +Visitor::Visitor(const vespalib::string & domain, TransLogClient & tlc, Callback & callBack) : + Session(domain, tlc), + _callback(callBack) +{ +} + +bool +Session::init(FRT_RPCRequest *req) +{ + int32_t retval(_tlc.rpc(req)); + req->SubRef(); + if (retval > 0) { + clear(); + _sessionId = retval; + SessionKey key(_domain, _sessionId); + { + LockGuard guard(_tlc._lock); + _tlc._sessions[key] = this; + } + retval = run(); + } + return (retval > 0); +} + +bool +Visitor::visit(const SerialNum & from, const SerialNum & to) +{ + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); + req->SetMethodName("domainVisit"); + req->GetParams()->AddString(_domain.c_str()); + req->GetParams()->AddInt64(from); + req->GetParams()->AddInt64(to); + return init(req); +} + +bool +Session::run() +{ + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); + req->SetMethodName("domainSessionRun"); + req->GetParams()->AddString(_domain.c_str()); + req->GetParams()->AddInt32(_sessionId); + int32_t retval(_tlc.rpc(req)); + req->SubRef(); + return (retval == 0); +} + +bool +Session::close() +{ + int retval(0); + if (_sessionId > 0) { + do { + FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); + req->SetMethodName("domainSessionClose"); + req->GetParams()->AddString(_domain.c_str()); + req->GetParams()->AddInt32(_sessionId); + if ( (retval = _tlc.rpc(req)) > 0) { + std::this_thread::sleep_for(10ms); + } + req->SubRef(); + } while ( retval == 1 ); + } + return (retval == 0); +} + +Visitor::~Visitor() = default; + +} diff --git a/searchlib/src/vespa/searchlib/transactionlog/client_session.h b/searchlib/src/vespa/searchlib/transactionlog/client_session.h new file mode 100644 index 00000000000..49f24d83aaf --- /dev/null +++ b/searchlib/src/vespa/searchlib/transactionlog/client_session.h @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "client_common.h" +#include <vespa/searchlib/common/serialnum.h> +#include <vespa/vespalib/util/buffer.h> +#include <vespa/vespalib/stllike/string.h> + +class FRT_RPCRequest; + +namespace search::transactionlog::client { + +class TransLogClient; + +class SessionKey +{ +public: + SessionKey(const vespalib::string & domain, int sessionId); + ~SessionKey(); + bool operator < (const SessionKey & b) const { return cmp(b) < 0; } +private: + int cmp(const SessionKey & b) const; + vespalib::string _domain; + int _sessionId; +}; + +class Session +{ +public: + Session(const vespalib::string & domain, TransLogClient & tlc); + virtual ~Session(); + /// You can commit data of any registered type to any channel. + bool commit(const vespalib::ConstBufferRef & packet); + /// Will erase all entries prior to <to> + bool erase(const SerialNum & to); + bool status(SerialNum & b, SerialNum & e, size_t & count); + + bool sync(const SerialNum &syncTo, SerialNum &syncedTo); + + virtual RPC::Result visit(const Packet & ) { return RPC::OK; } + virtual void eof() { } + bool close(); + void clear(); + const vespalib::string & getDomain() const { return _domain; } + const TransLogClient & getTLC() const { return _tlc; } +protected: + bool init(FRT_RPCRequest * req); + bool run(); + TransLogClient & _tlc; + vespalib::string _domain; + int _sessionId; +}; + +/// Here you connect to the incomming data getting everything from <from> +class Visitor : public Session +{ +public: + Visitor(const vespalib::string & domain, TransLogClient & tlc, Callback & callBack); + bool visit(const SerialNum & from, const SerialNum & to); + virtual ~Visitor() override; + RPC::Result visit(const Packet & packet) override { return _callback.receive(packet); } + void eof() override { _callback.eof(); } +private: + Callback & _callback; +}; + +} + diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index c5427c5b401..8bfa17cfcfb 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -10,14 +10,7 @@ namespace search::transactionlog { /// This represents a type of the entry. Fx update,remove -typedef uint32_t Type; -/// A channel represents one data stream. - -class RPC -{ -public: -enum Result { OK, FULL, ERROR }; -}; +using Type = uint32_t; class SerialNumRange { @@ -92,7 +85,13 @@ class Writer { public: using DoneCallback = std::shared_ptr<IDestructorCallback>; virtual ~Writer() = default; - virtual void commit(const vespalib::string & domainName, const Packet & packet, DoneCallback done) = 0; + virtual void commit(const Packet & packet, DoneCallback done) = 0; +}; + +class WriterFactory { +public: + virtual ~WriterFactory() = default; + virtual std::shared_ptr<Writer> getWriter(const vespalib::string & domainName) const = 0; }; class Destination { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index c2cdb6f86b9..9adff564cc8 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -12,7 +12,7 @@ namespace search::transactionlog { class DomainPart; class Session; -class Domain +class Domain : public Writer { public: using SP = std::shared_ptr<Domain>; @@ -21,13 +21,13 @@ public: Domain(const vespalib::string &name, const vespalib::string &baseDir, Executor & executor, const DomainConfig & cfg, const common::FileHeaderContext &fileHeaderContext); - ~Domain(); + ~Domain() override; DomainInfo getDomainInfo() const; const vespalib::string & name() const { return _name; } bool erase(SerialNum to); - void commit(const Packet & packet, Writer::DoneCallback onDone); + void commit(const Packet & packet, Writer::DoneCallback onDone) override; int visit(const Domain::SP & self, SerialNum from, SerialNum to, std::unique_ptr<Destination> dest); SerialNum begin() const; diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp index 2c6c1e249f4..84919a59a97 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp @@ -1,12 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "translogclient.h" +#include "common.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/transport.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/fastos/thread.h> -#include <thread> + #include <vespa/log/log.h> LOG_SETUP(".translogclient"); @@ -15,7 +17,7 @@ using namespace std::chrono_literals; VESPA_THREAD_STACK_TAG(translogclient_rpc_callback) -namespace search::transactionlog { +namespace search::transactionlog::client { namespace { const double NEVER(-1.0); @@ -33,7 +35,7 @@ struct RpcTask : public vespalib::Executor::Task { req->Return(); req = nullptr; } - ~RpcTask() { + ~RpcTask() override { if (req != nullptr) { req->SetError(FRTE_RPC_METHOD_FAILED, "client has been shut down"); req->Return(); @@ -46,13 +48,13 @@ struct RpcTask : public vespalib::Executor::Task { using vespalib::LockGuard; TransLogClient::TransLogClient(const vespalib::string & rpcTarget) : - _executor(1, 128 * 1024, translogclient_rpc_callback), + _executor(std::make_unique<vespalib::ThreadStackExecutor>(1, 128 * 1024, translogclient_rpc_callback)), _rpcTarget(rpcTarget), _sessions(), _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)), _transport(std::make_unique<FNET_Transport>()), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())), - _target(NULL) + _target(nullptr) { reconnect(); exportRPC(*_supervisor); @@ -62,29 +64,33 @@ TransLogClient::TransLogClient(const vespalib::string & rpcTarget) : TransLogClient::~TransLogClient() { disconnect(); - _executor.shutdown().sync(); + _executor->shutdown().sync(); _transport->ShutDown(true); } -bool TransLogClient::reconnect() +bool +TransLogClient::reconnect() { disconnect(); _target = _supervisor->Get2WayTarget(_rpcTarget.c_str()); return isConnected(); } -bool TransLogClient::isConnected() const { - return (_target != NULL) && _target->IsValid(); +bool +TransLogClient::isConnected() const { + return (_target != nullptr) && _target->IsValid(); } -void TransLogClient::disconnect() +void +TransLogClient::disconnect() { if (_target) { _target->SubRef(); } } -bool TransLogClient::create(const vespalib::string & domain) +bool +TransLogClient::create(const vespalib::string & domain) { FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("createDomain"); @@ -94,7 +100,8 @@ bool TransLogClient::create(const vespalib::string & domain) return (retval == 0); } -bool TransLogClient::remove(const vespalib::string & domain) +bool +TransLogClient::remove(const vespalib::string & domain) { FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("deleteDomain"); @@ -104,27 +111,28 @@ bool TransLogClient::remove(const vespalib::string & domain) return (retval == 0); } -TransLogClient::Session::UP TransLogClient::open(const vespalib::string & domain) +std::unique_ptr<Session> +TransLogClient::open(const vespalib::string & domain) { - Session::UP session; FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("openDomain"); req->GetParams()->AddString(domain.c_str()); int32_t retval(rpc(req)); + req->SubRef(); if (retval == 0) { - session.reset(new Session(domain, *this)); + return std::make_unique<Session>(domain, *this); } - req->SubRef(); - return session; + return std::unique_ptr<Session>(); } -TransLogClient::Visitor::UP -TransLogClient::createVisitor(const vespalib::string & domain, TransLogClient::Session::Callback & callBack) +std::unique_ptr<Visitor> +TransLogClient::createVisitor(const vespalib::string & domain, Callback & callBack) { - return TransLogClient::Visitor::UP(new Visitor(domain, *this, callBack)); + return std::make_unique<Visitor>(domain, *this, callBack); } -bool TransLogClient::listDomains(std::vector<vespalib::string> & dir) +bool +TransLogClient::listDomains(std::vector<vespalib::string> & dir) { FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); req->SetMethodName("listDomains"); @@ -139,7 +147,8 @@ bool TransLogClient::listDomains(std::vector<vespalib::string> & dir) return (retval == 0); } -int32_t TransLogClient::rpc(FRT_RPCRequest * req) +int32_t +TransLogClient::rpc(FRT_RPCRequest * req) { int32_t retval(-7); if (_target) { @@ -156,15 +165,17 @@ int32_t TransLogClient::rpc(FRT_RPCRequest * req) return retval; } -TransLogClient::Session * TransLogClient::findSession(const vespalib::string & domainName, int sessionId) +Session * +TransLogClient::findSession(const vespalib::string & domainName, int sessionId) { SessionKey key(domainName, sessionId); SessionMap::iterator found(_sessions.find(key)); - Session * session((found != _sessions.end()) ? found->second : NULL); + Session * session((found != _sessions.end()) ? found->second : nullptr); return session; } -void TransLogClient::exportRPC(FRT_Supervisor & supervisor) +void +TransLogClient::exportRPC(FRT_Supervisor & supervisor) { FRT_ReflectionBuilder rb( & supervisor); @@ -185,7 +196,8 @@ void TransLogClient::exportRPC(FRT_Supervisor & supervisor) } -void TransLogClient::do_visitCallbackRPC(FRT_RPCRequest *req) +void +TransLogClient::do_visitCallbackRPC(FRT_RPCRequest *req) { uint32_t retval(uint32_t(-1)); FRT_Values & params = *req->GetParams(); @@ -194,7 +206,7 @@ void TransLogClient::do_visitCallbackRPC(FRT_RPCRequest *req) int32_t sessionId(params[1]._intval32); LOG(spam, "visitCallback(%s, %d)(%d)", domainName, sessionId, params[2]._data._len); Session * session(findSession(domainName, sessionId)); - if (session != NULL) { + if (session != nullptr) { Packet packet(params[2]._data._buf, params[2]._data._len); retval = session->visit(packet); } @@ -202,7 +214,8 @@ void TransLogClient::do_visitCallbackRPC(FRT_RPCRequest *req) LOG(debug, "visitCallback(%s, %d)=%d done", domainName, sessionId, retval); } -void TransLogClient::do_eofCallbackRPC(FRT_RPCRequest *req) +void +TransLogClient::do_eofCallbackRPC(FRT_RPCRequest *req) { uint32_t retval(uint32_t(-1)); FRT_Values & params = *req->GetParams(); @@ -211,7 +224,7 @@ void TransLogClient::do_eofCallbackRPC(FRT_RPCRequest *req) int32_t sessionId(params[1]._intval32); LOG(debug, "eofCallback(%s, %d)", domainName, sessionId); Session * session(findSession(domainName, sessionId)); - if (session != NULL) { + if (session != nullptr) { session->eof(); retval = 0; } @@ -219,183 +232,16 @@ void TransLogClient::do_eofCallbackRPC(FRT_RPCRequest *req) LOG(debug, "eofCallback(%s, %d)=%d done", domainName, sessionId, retval); } -void TransLogClient::visitCallbackRPC_hook(FRT_RPCRequest *req) -{ - _executor.execute(std::make_unique<RpcTask>(req->Detach(), [this](FRT_RPCRequest *x){ do_visitCallbackRPC(x); })); -} - -void TransLogClient::eofCallbackRPC_hook(FRT_RPCRequest *req) -{ - _executor.execute(std::make_unique<RpcTask>(req->Detach(), [this](FRT_RPCRequest *x){ do_eofCallbackRPC(x); })); -} - - -TransLogClient::Session::Session(const vespalib::string & domain, TransLogClient & tlc) : - _tlc(tlc), - _domain(domain), - _sessionId(0) -{ -} - -TransLogClient::Session::~Session() -{ - close(); - clear(); -} - -bool TransLogClient::Session::commit(const vespalib::ConstBufferRef & buf) -{ - bool retval(true); - if (buf.size() != 0) { - FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); - req->SetMethodName("domainCommit"); - req->GetParams()->AddString(_domain.c_str()); - req->GetParams()->AddData(buf.c_str(), buf.size()); - int retcode = _tlc.rpc(req); - retval = (retcode == 0); - if (retval) { - req->SubRef(); - } else { - vespalib::string msg; - if (req->GetReturn() != 0) { - msg = req->GetReturn()->GetValue(1)._string._str; - } else { - msg = vespalib::make_string("Clientside error %s: error(%d): %s", req->GetMethodName(), req->GetErrorCode(), req->GetErrorMessage()); - } - req->SubRef(); - throw std::runtime_error(vespalib::make_string("commit failed with code %d. server says: %s", retcode, msg.c_str())); - } - } - return retval; -} - -bool TransLogClient::Session::status(SerialNum & b, SerialNum & e, size_t & count) -{ - FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); - req->SetMethodName("domainStatus"); - req->GetParams()->AddString(_domain.c_str()); - int32_t retval(_tlc.rpc(req)); - if (retval == 0) { - b = req->GetReturn()->GetValue(1)._intval64; - e = req->GetReturn()->GetValue(2)._intval64; - count = req->GetReturn()->GetValue(3)._intval64; - } - req->SubRef(); - return (retval == 0); -} - -bool TransLogClient::Session::erase(const SerialNum & to) -{ - FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); - req->SetMethodName("domainPrune"); - req->GetParams()->AddString(_domain.c_str()); - req->GetParams()->AddInt64(to); - int32_t retval(_tlc.rpc(req)); - req->SubRef(); - if (retval == 1) { - LOG(warning, "Prune to %" PRIu64 " denied since there were active visitors in that area", to); - } - return (retval == 0); -} - - -bool -TransLogClient::Session::sync(const SerialNum &syncTo, SerialNum &syncedTo) -{ - FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); - req->SetMethodName("domainSync"); - FRT_Values & params = *req->GetParams(); - params.AddString(_domain.c_str()); - params.AddInt64(syncTo); - int32_t retval(_tlc.rpc(req)); - if (retval == 0) { - syncedTo = req->GetReturn()->GetValue(1)._intval64; - } - req->SubRef(); - return (retval == 0); -} - - -void TransLogClient::Session::clear() -{ - if (_sessionId > 0) { - LockGuard guard(_tlc._lock); - _tlc._sessions.erase(SessionKey(_domain, _sessionId)); - } - _sessionId = 0; -} - -int TransLogClient::SessionKey::cmp(const TransLogClient::SessionKey & b) const +void +TransLogClient::visitCallbackRPC_hook(FRT_RPCRequest *req) { - int diff(strcmp(_domain.c_str(), b._domain.c_str())); - if (diff == 0) { - diff = _sessionId - b._sessionId; - } - return diff; + _executor->execute(std::make_unique<RpcTask>(req->Detach(), [this](FRT_RPCRequest *x){ do_visitCallbackRPC(x); })); } -TransLogClient::Visitor::Visitor(const vespalib::string & domain, TransLogClient & tlc, Callback & callBack) : - Session(domain, tlc), - _callback(callBack) +void +TransLogClient::eofCallbackRPC_hook(FRT_RPCRequest *req) { + _executor->execute(std::make_unique<RpcTask>(req->Detach(), [this](FRT_RPCRequest *x){ do_eofCallbackRPC(x); })); } -bool TransLogClient::Session::init(FRT_RPCRequest *req) -{ - int32_t retval(_tlc.rpc(req)); - req->SubRef(); - if (retval > 0) { - clear(); - _sessionId = retval; - SessionKey key(_domain, _sessionId); - { - LockGuard guard(_tlc._lock); - _tlc._sessions[key] = this; - } - retval = run(); - } - return (retval > 0); -} - -bool TransLogClient::Visitor::visit(const SerialNum & from, const SerialNum & to) -{ - FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); - req->SetMethodName("domainVisit"); - req->GetParams()->AddString(_domain.c_str()); - req->GetParams()->AddInt64(from); - req->GetParams()->AddInt64(to); - return init(req); -} - -bool TransLogClient::Session::run() -{ - FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); - req->SetMethodName("domainSessionRun"); - req->GetParams()->AddString(_domain.c_str()); - req->GetParams()->AddInt32(_sessionId); - int32_t retval(_tlc.rpc(req)); - req->SubRef(); - return (retval == 0); -} - -bool TransLogClient::Session::close() -{ - int retval(0); - if (_sessionId > 0) { - do { - FRT_RPCRequest *req = _tlc._supervisor->AllocRPCRequest(); - req->SetMethodName("domainSessionClose"); - req->GetParams()->AddString(_domain.c_str()); - req->GetParams()->AddInt32(_sessionId); - if ( (retval = _tlc.rpc(req)) > 0) { - std::this_thread::sleep_for(10ms); - } - req->SubRef(); - } while ( retval == 1 ); - } - return (retval == 0); -} - -TransLogClient::Visitor::~Visitor() = default; - } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h index 38c30cd5b4c..289a0fcb8c0 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h @@ -1,11 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "common.h" -#include <vespa/document/util/bytebuffer.h> +#include "client_common.h" +#include "client_session.h" #include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/util/buffer.h> -#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/fnet/frt/invokable.h> #include <map> #include <vector> @@ -13,90 +11,39 @@ class FNET_Transport; class FRT_Supervisor; class FRT_Target; +class FastOS_ThreadPool; -namespace search::transactionlog { +namespace vespalib { class ThreadStackExecutorBase; } +namespace search::transactionlog::client { + +class Session; +class Visitor; class TransLogClient : private FRT_Invokable { -private: - TransLogClient(const TransLogClient &); - TransLogClient& operator=(const TransLogClient &); - public: - class Session - { - public: - class Callback { - public: - virtual ~Callback() { } - virtual RPC::Result receive(const Packet & packet) = 0; - virtual void eof() { } - }; - public: - typedef std::unique_ptr<Session> UP; - typedef std::shared_ptr<Session> SP; - - Session(const vespalib::string & domain, TransLogClient & tlc); - virtual ~Session(); - /// You can commit data of any registered type to any channel. - bool commit(const vespalib::ConstBufferRef & packet); - /// Will erase all entries prior to <to> - bool erase(const SerialNum & to); - bool status(SerialNum & b, SerialNum & e, size_t & count); - - bool sync(const SerialNum &syncTo, SerialNum &syncedTo); - - virtual RPC::Result visit(const Packet & ) { return RPC::OK; } - virtual void eof() { } - bool close(); - void clear(); - const vespalib::string & getDomain() const { return _domain; } - const TransLogClient & getTLC() const { return _tlc; } - protected: - bool init(FRT_RPCRequest * req); - bool run(); - TransLogClient & _tlc; - vespalib::string _domain; - int _sessionId; - }; - /// Here you connect to the incomming data getting everything from <from> - class Visitor : public Session - { - public: - typedef std::unique_ptr<Visitor> UP; - typedef std::shared_ptr<Visitor> SP; - - Visitor(const vespalib::string & domain, TransLogClient & tlc, Callback & callBack); - bool visit(const SerialNum & from, const SerialNum & to); - virtual ~Visitor(); - RPC::Result visit(const Packet & packet) override { return _callback.receive(packet); } - void eof() override { _callback.eof(); } - private: - Callback & _callback; - }; - /// Here you read the incomming data getting everything from <from> - -public: - typedef std::unique_ptr<TransLogClient> UP; - TransLogClient(const vespalib::string & rpctarget); - virtual ~TransLogClient(); + TransLogClient(const TransLogClient &) = delete; + TransLogClient& operator=(const TransLogClient &) = delete; + ~TransLogClient() override; /// Here you create a new domain bool create(const vespalib::string & domain); /// Here you remove a domain bool remove(const vespalib::string & domain); /// Here you open an existing domain - Session::UP open(const vespalib::string & domain); + std::unique_ptr<Session> open(const vespalib::string & domain); /// Here you can get a list of available domains. bool listDomains(std::vector<vespalib::string> & dir); - Visitor::UP createVisitor(const vespalib::string & domain, Session::Callback & callBack); + std::unique_ptr<Visitor> createVisitor(const vespalib::string & domain, Callback & callBack); bool isConnected() const; void disconnect(); bool reconnect(); const vespalib::string &getRPCTarget() const { return _rpcTarget; } private: + friend Session; + friend Visitor; void exportRPC(FRT_Supervisor & supervisor); void do_visitCallbackRPC(FRT_RPCRequest *req); void do_eofCallbackRPC(FRT_RPCRequest *req); @@ -105,22 +52,11 @@ private: int32_t rpc(FRT_RPCRequest * req); Session * findSession(const vespalib::string & domain, int sessionId); - class SessionKey - { - public: - SessionKey(const vespalib::string & domain, int sessionId) : _domain(domain), _sessionId(sessionId) { } - bool operator < (const SessionKey & b) const { return cmp(b) < 0; } - private: - int cmp(const SessionKey & b) const; - vespalib::string _domain; - int _sessionId; - }; - - typedef std::map< SessionKey, Session * > SessionMap; + using SessionMap = std::map< SessionKey, Session * >; - vespalib::ThreadStackExecutor _executor; - vespalib::string _rpcTarget; - SessionMap _sessions; + std::unique_ptr<vespalib::ThreadStackExecutorBase> _executor; + vespalib::string _rpcTarget; + SessionMap _sessions; //Brute force lock for subscriptions. For multithread safety. vespalib::Lock _lock; std::unique_ptr<FastOS_ThreadPool> _threadPool; diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 42bf057d3cf..fcef06c3f2e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "translogserver.h" #include "domain.h" +#include "client_common.h" #include <vespa/searchlib/common/gatecallback.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/io/fileutil.h> @@ -205,13 +206,13 @@ TransLogServer::run() vespalib::duration TransLogServer::getChunkAgeLimit() const { - Guard domainGuard(_domainMutex); + ReadGuard domainGuard(_domainMutex); return _domainConfig.getChunkAgeLimit(); } TransLogServer & TransLogServer::setDomainConfig(const DomainConfig & cfg) { - Guard domainGuard(_domainMutex); + WriteGuard domainGuard(_domainMutex); _domainConfig = cfg; for(auto &domain: _domains) { domain.second->setConfig(cfg); @@ -223,7 +224,7 @@ DomainStats TransLogServer::getDomainStats() const { DomainStats retval; - Guard domainGuard(_domainMutex); + ReadGuard domainGuard(_domainMutex); for (const auto &elem : _domains) { retval[elem.first] = elem.second->getDomainInfo(); } @@ -234,7 +235,7 @@ std::vector<vespalib::string> TransLogServer::getDomainNames() { std::vector<vespalib::string> names; - Guard guard(_domainMutex); + ReadGuard guard(_domainMutex); for(const auto &domain: _domains) { names.push_back(domain.first); } @@ -242,15 +243,14 @@ TransLogServer::getDomainNames() } Domain::SP -TransLogServer::findDomain(stringref domainName) +TransLogServer::findDomain(stringref domainName) const { - Guard domainGuard(_domainMutex); - Domain::SP domain; - DomainList::iterator found(_domains.find(domainName)); + ReadGuard domainGuard(_domainMutex); + auto found(_domains.find(domainName)); if (found != _domains.end()) { - domain = found->second; + return found->second; } - return domain; + return DomainSP(); } void @@ -346,10 +346,10 @@ namespace { constexpr double NEVER(-1.0); void -writeDomainDir(std::lock_guard<std::mutex> &guard, +writeDomainDir(std::shared_lock<std::shared_mutex> &guard, vespalib::string dir, vespalib::string domainList, - std::map<vespalib::string, std::shared_ptr<Domain>> &domains) + const std::map<vespalib::string, std::shared_ptr<Domain>> &domains) { (void) guard; vespalib::string domainListTmp(domainList + ".tmp"); @@ -400,12 +400,12 @@ public: private: bool send(FRT_RPCRequest * req) { int32_t retval = rpc(req); - if ( ! ((retval == RPC::OK) || (retval == FRTE_RPC_CONNECTION)) ) { + if ( ! ((retval == client::RPC::OK) || (retval == FRTE_RPC_CONNECTION)) ) { LOG(error, "Return value != OK(%d) in send for method 'visitCallback'.", retval); } req->SubRef(); - return (retval == RPC::OK); + return (retval == client::RPC::OK); } int32_t rpc(FRT_RPCRequest * req) { int32_t retval(-7); @@ -443,13 +443,16 @@ TransLogServer::createDomain(FRT_RPCRequest *req) const char * domainName = params[0]._string._str; LOG(debug, "createDomain(%s)", domainName); - Guard createDeleteGuard(_fileLock); + std::lock_guard createDeleteGuard(_fileLock); Domain::SP domain(findDomain(domainName)); if ( !domain ) { try { domain = std::make_shared<Domain>(domainName, dir(), _executor, _domainConfig, _fileHeaderContext); - Guard domainGuard(_domainMutex); - _domains[domain->name()] = domain; + { + WriteGuard domainGuard(_domainMutex); + _domains[domain->name()] = domain; + } + ReadGuard domainGuard(_domainMutex); writeDomainDir(domainGuard, dir(), domainList(), _domains); } catch (const std::exception & e) { LOG(warning, "Failed creating %s domain. Exception = %s", domainName, e.what()); @@ -471,18 +474,18 @@ TransLogServer::deleteDomain(FRT_RPCRequest *req) const char * domainName = params[0]._string._str; LOG(debug, "deleteDomain(%s)", domainName); - Guard createDeleteGuard(_fileLock); + std::lock_guard createDeleteGuard(_fileLock); Domain::SP domain(findDomain(domainName)); if ( !domain || (domain->getNumSessions() == 0)) { try { if (domain) { domain->markDeleted(); - Guard domainGuard(_domainMutex); + WriteGuard domainGuard(_domainMutex); _domains.erase(domainName); } vespalib::rmdir(Domain::getDir(dir(), domainName), true); vespalib::File::sync(dir()); - Guard domainGuard(_domainMutex); + ReadGuard domainGuard(_domainMutex); writeDomainDir(domainGuard, dir(), domainList(), _domains); } catch (const std::exception & e) { msg = make_string("Failed deleting %s domain. Exception = %s", domainName, e.what()); @@ -523,7 +526,7 @@ TransLogServer::listDomains(FRT_RPCRequest *req) LOG(debug, "listDomains()"); vespalib::string domains; - Guard domainGuard(_domainMutex); + ReadGuard domainGuard(_domainMutex); for(DomainList::const_iterator it(_domains.begin()), mt(_domains.end()); it != mt; it++) { domains += it->second->name(); domains += "\n"; @@ -553,12 +556,12 @@ TransLogServer::domainStatus(FRT_RPCRequest *req) } } -void -TransLogServer::commit(const vespalib::string & domainName, const Packet & packet, DoneCallback done) +std::shared_ptr<Writer> +TransLogServer::getWriter(const vespalib::string & domainName) const { Domain::SP domain(findDomain(domainName)); if (domain) { - domain->commit(packet, std::move(done)); + return domain; } else { throw IllegalArgumentException("Could not find domain " + domainName); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index ed5d475bb73..77e38c90c9a 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -6,7 +6,7 @@ #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/document/util/queue.h> #include <vespa/fnet/frt/invokable.h> -#include <mutex> +#include <shared_mutex> class FRT_Supervisor; class FNET_Transport; @@ -18,7 +18,7 @@ namespace search::transactionlog { class TransLogServerExplorer; class Domain; -class TransLogServer : public document::Runnable, private FRT_Invokable, public Writer +class TransLogServer : public document::Runnable, private FRT_Invokable, public WriterFactory { public: friend class TransLogServerExplorer; @@ -33,7 +33,7 @@ public: ~TransLogServer() override; DomainStats getDomainStats() const; bool commitIfStale(); - void commit(const vespalib::string & domainName, const Packet & packet, DoneCallback done) override; + std::shared_ptr<Writer> getWriter(const vespalib::string & domainName) const override; TransLogServer & setDomainConfig(const DomainConfig & cfg); vespalib::duration getChunkAgeLimit() const; @@ -72,13 +72,15 @@ private: void downSession(FRT_RPCRequest *req); std::vector<vespalib::string> getDomainNames(); - DomainSP findDomain(vespalib::stringref name); + DomainSP findDomain(vespalib::stringref name) const; vespalib::string dir() const { return _baseDir + "/" + _name; } vespalib::string domainList() const { return dir() + "/" + _name + ".domains"; } static const Session::SP & getSession(FRT_RPCRequest *req); using DomainList = std::map<vespalib::string, DomainSP >; + using ReadGuard = std::shared_lock<std::shared_mutex>; + using WriteGuard = std::unique_lock<std::shared_mutex>; vespalib::string _name; vespalib::string _baseDir; @@ -88,13 +90,11 @@ private: std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _supervisor; DomainList _domains; - mutable std::mutex _domainMutex; // Protects _domains + mutable std::shared_mutex _domainMutex;; // Protects _domains std::condition_variable _domainCondition; std::mutex _fileLock; // Protects the creating and deleting domains including file system operations. document::Queue<FRT_RPCRequest *> _reqQ; const common::FileHeaderContext &_fileHeaderContext; - using Guard = std::lock_guard<std::mutex>; - using MonitorGuard = std::unique_lock<std::mutex>; }; } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java index 985685ebb8d..5813e2cef39 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthUpdater.java @@ -8,11 +8,18 @@ import java.net.URL; import java.time.Duration; import java.time.Instant; import java.util.Optional; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author hakonhall */ class StateV1HealthUpdater implements HealthUpdater { + private static final Logger logger = Logger.getLogger(StateV1HealthUpdater.class.getName()); + + private static final Pattern CONFIG_SERVER_ENDPOINT_PATTERN = Pattern.compile("^http://(cfg[0-9]+)\\."); + private final String endpoint; private final StateV1HealthClient healthClient; @@ -46,8 +53,18 @@ class StateV1HealthUpdater implements HealthUpdater { } ServiceStatus newServiceStatus = healthInfo.isHealthy() ? ServiceStatus.UP : ServiceStatus.DOWN; - Optional<Instant> newSince = newServiceStatus == serviceStatusInfo.serviceStatus() ? - serviceStatusInfo.since() : Optional.of(now); + + final Optional<Instant> newSince; + if (newServiceStatus == serviceStatusInfo.serviceStatus()) { + newSince = serviceStatusInfo.since(); + } else { + newSince = Optional.of(now); + + Matcher matcher = CONFIG_SERVER_ENDPOINT_PATTERN.matcher(endpoint); + if (matcher.find()) { + logger.info("New health status for " + matcher.group(1) + ": " + healthInfo.toString()); + } + } serviceStatusInfo = new ServiceStatusInfo(newServiceStatus, newSince, Optional.of(now), healthInfo.getErrorDescription(), Optional.of(endpoint)); diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index dc0a0194d36..426f57db21b 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -39,6 +39,7 @@ vespa_define_module( src/vespa/storage/persistence src/vespa/storage/persistence/filestorage src/vespa/storage/storageserver + src/vespa/storage/storageserver/rpc src/vespa/storage/storageutil src/vespa/storage/tools src/vespa/storage/visiting @@ -60,5 +61,6 @@ vespa_define_module( src/tests/persistence/common src/tests/persistence/filestorage src/tests/storageserver + src/tests/storageserver/rpc src/tests/visiting ) diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index e49a69414b7..b3c2257c3d9 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_executable(storage_storageserver_gtest_runner_app TEST communicationmanagertest.cpp configurable_bucket_resolver_test.cpp documentapiconvertertest.cpp - fnet_listener_test.cpp mergethrottlertest.cpp priorityconvertertest.cpp service_layer_error_listener_test.cpp diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index 0f95f7e7ece..1431c49559c 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -167,7 +167,7 @@ TEST_F(CommunicationManagerTest, commands_are_dequeued_in_fifo_order) { // Lower number == higher priority. std::vector<api::StorageMessage::Priority> pris{200, 0, 255, 128}; for (auto pri : pris) { - storage.enqueue(createDummyCommand(pri)); + storage.dispatch_async(createDummyCommand(pri)); } storageLink->waitForMessages(pris.size(), MESSAGE_WAIT_TIME_SEC); @@ -195,7 +195,7 @@ TEST_F(CommunicationManagerTest, replies_are_dequeued_in_fifo_order) { std::vector<api::StorageMessage::Priority> pris{200, 0, 255, 128}; for (auto pri : pris) { - storage.enqueue(createDummyCommand(pri)->makeReply()); + storage.dispatch_async(createDummyCommand(pri)->makeReply()); } storageLink->waitForMessages(pris.size(), MESSAGE_WAIT_TIME_SEC); diff --git a/storage/src/tests/storageserver/rpc/.gitignore b/storage/src/tests/storageserver/rpc/.gitignore new file mode 100644 index 00000000000..a04ce7d5907 --- /dev/null +++ b/storage/src/tests/storageserver/rpc/.gitignore @@ -0,0 +1 @@ +storage_storageserver_rpc_gtest_runner_app diff --git a/storage/src/tests/storageserver/rpc/CMakeLists.txt b/storage/src/tests/storageserver/rpc/CMakeLists.txt new file mode 100644 index 00000000000..808ad8711ee --- /dev/null +++ b/storage/src/tests/storageserver/rpc/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(storage_storageserver_rpc_gtest_runner_app TEST + SOURCES + caching_rpc_target_resolver_test.cpp + cluster_controller_rpc_api_service_test.cpp + message_codec_provider_test.cpp + storage_api_rpc_service_test.cpp + gtest_runner.cpp + DEPENDS + storage_storageserver + storage_testcommon + storage_teststorageserver + GTest::GTest +) + +vespa_add_test( + NAME storage_storageserver_rpc_gtest_runner_app + COMMAND storage_storageserver_rpc_gtest_runner_app +) diff --git a/storage/src/tests/storageserver/rpc/caching_rpc_target_resolver_test.cpp b/storage/src/tests/storageserver/rpc/caching_rpc_target_resolver_test.cpp new file mode 100644 index 00000000000..9a2eb4d5c64 --- /dev/null +++ b/storage/src/tests/storageserver/rpc/caching_rpc_target_resolver_test.cpp @@ -0,0 +1,134 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/slobrok/imirrorapi.h> +#include <vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/vdslib/state/nodetype.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace storage::rpc; +using slobrok::api::IMirrorAPI; +using storage::api::StorageMessageAddress; +using storage::lib::NodeType; + +class MockMirror : public IMirrorAPI { +public: + using Mappings = std::map<vespalib::string, IMirrorAPI::SpecList>; + Mappings mappings; + uint32_t gen; + MockMirror() : mappings(), gen(1) {} + SpecList lookup(const std::string& pattern) const override { + auto itr = mappings.find(pattern); + if (itr != mappings.end()) { + return itr->second; + } + return {}; + } + uint32_t updates() const override { return gen; } + bool ready() const override { return true; } + void inc_gen() { ++gen; } +}; + +class MockWrappedFrtTarget : public WrappedFrtTarget { +private: + bool& _valid; +public: + MockWrappedFrtTarget(bool& valid) : _valid(valid) {} + FRT_Target* get() noexcept override { return nullptr; } + bool is_valid() const noexcept override { return _valid; } +}; + +class MockTargetFactory : public RpcTargetFactory { +public: + mutable bool valid_target; + + MockTargetFactory() : valid_target(true) {} + std::unique_ptr<RpcTarget> make_target(const vespalib::string& connection_spec, uint32_t slobrok_gen) const override { + return std::make_unique<RpcTarget>(std::make_unique<MockWrappedFrtTarget>(valid_target), + connection_spec, slobrok_gen); + } +}; + +class CachingRpcTargetResolverTest : public ::testing::Test { +public: + MockMirror mirror; + MockTargetFactory factory; + CachingRpcTargetResolver resolver; + StorageMessageAddress address_0; + StorageMessageAddress address_1; + vespalib::string spec_0; + vespalib::string spec_1; + + CachingRpcTargetResolverTest() + : mirror(), + factory(), + resolver(mirror, factory), + address_0("my_cluster", NodeType::STORAGE, 5), + address_1("my_cluster", NodeType::DISTRIBUTOR, 7), + spec_0("tcp/my:41"), + spec_1("tcp/my:42") + { + add_mapping(address_0, spec_0); + } + void add_mapping(const StorageMessageAddress& address, const vespalib::string& connection_spec) { + mirror.mappings[to_slobrok_id(address)] = {{to_slobrok_id(address), connection_spec}}; + } + static vespalib::string to_slobrok_id(const storage::api::StorageMessageAddress& address) { + return CachingRpcTargetResolver::address_to_slobrok_id(address); + } +}; + +TEST_F(CachingRpcTargetResolverTest, converts_storage_message_address_to_slobrok_id) +{ + EXPECT_EQ("storage/cluster.my_cluster/storage/5", to_slobrok_id(address_0)); + EXPECT_EQ("storage/cluster.my_cluster/distributor/7", to_slobrok_id(address_1)); +} + +TEST_F(CachingRpcTargetResolverTest, resolves_rpc_target_and_caches_result) +{ + auto target_a = resolver.resolve_rpc_target(address_0); + ASSERT_TRUE(target_a); + EXPECT_EQ(spec_0, target_a->_spec); + EXPECT_EQ(1, target_a->_slobrok_gen); + auto target_b = resolver.resolve_rpc_target(address_0); + ASSERT_TRUE(target_b); + EXPECT_EQ(target_a.get(), target_b.get()); + EXPECT_EQ(spec_0, target_b->_spec); + EXPECT_EQ(1, target_b->_slobrok_gen); +} + +TEST_F(CachingRpcTargetResolverTest, cached_rpc_target_is_updated_when_slobrok_generation_changes) +{ + auto target_a = resolver.resolve_rpc_target(address_0); + mirror.inc_gen(); + auto target_b = resolver.resolve_rpc_target(address_0); + EXPECT_EQ(target_a.get(), target_b.get()); + EXPECT_EQ(2, target_b->_slobrok_gen); +} + +TEST_F(CachingRpcTargetResolverTest, new_rpc_target_is_created_if_connection_spec_changes) +{ + auto target_a = resolver.resolve_rpc_target(address_0); + add_mapping(address_0, spec_1); + mirror.inc_gen(); + auto target_b = resolver.resolve_rpc_target(address_0); + EXPECT_NE(target_a.get(), target_b.get()); + EXPECT_EQ(spec_1, target_b->_spec); + EXPECT_EQ(2, target_b->_slobrok_gen); +} + +TEST_F(CachingRpcTargetResolverTest, new_rpc_target_is_created_if_raw_target_is_invalid) +{ + auto target_a = resolver.resolve_rpc_target(address_0); + factory.valid_target = false; + auto target_b = resolver.resolve_rpc_target(address_0); + EXPECT_NE(target_a.get(), target_b.get()); + EXPECT_EQ(spec_0, target_b->_spec); + EXPECT_EQ(1, target_b->_slobrok_gen); +} + +TEST_F(CachingRpcTargetResolverTest, null_rpc_target_is_returned_if_slobrok_id_is_not_found) +{ + auto target = resolver.resolve_rpc_target(address_1); + EXPECT_FALSE(target); +} diff --git a/storage/src/tests/storageserver/fnet_listener_test.cpp b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp index f82af1f8e5c..09c9ddb1f72 100644 --- a/storage/src/tests/storageserver/fnet_listener_test.cpp +++ b/storage/src/tests/storageserver/rpc/cluster_controller_rpc_api_service_test.cpp @@ -1,33 +1,37 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/storage/storageserver/fnetlistener.h> -#include <vespa/storage/storageserver/message_enqueuer.h> +#include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/storage/storageserver/message_dispatcher.h> #include <vespa/storage/storageserver/rpcrequestwrapper.h> -#include <vespa/storage/storageserver/slime_cluster_state_bundle_codec.h> +#include <vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h> +#include <vespa/storage/storageserver/rpc/shared_rpc_resources.h> +#include <vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.h> #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vdstestlib/config/dirconfig.hpp> -#include <vespa/messagebus/testlib/slobrok.h> #include <tests/common/testhelper.h> #include <vespa/vespalib/gtest/gtest.h> #include <vector> -namespace storage { +namespace storage::rpc { using document::FixedBucketSpaces; using namespace ::testing; -struct FNetListenerTest : Test { +struct ClusterControllerApiRpcServiceTest : Test { }; namespace { -struct MockOperationEnqueuer : MessageEnqueuer { +struct MockOperationDispatcher : MessageDispatcher { std::vector<std::shared_ptr<api::StorageMessage>> _enqueued; - void enqueue(std::shared_ptr<api::StorageMessage> msg) override { + void dispatch_sync(std::shared_ptr<api::StorageMessage> msg) override { + _enqueued.emplace_back(std::move(msg)); + } + void dispatch_async(std::shared_ptr<api::StorageMessage> msg) override { _enqueued.emplace_back(std::move(msg)); } }; @@ -38,11 +42,11 @@ struct DummyReturnHandler : FRT_IReturnHandler { }; struct FixtureBase { - // TODO factor out Slobrok code to avoid need to set up live ports for unrelated tests mbus::Slobrok slobrok; vdstestlib::DirConfig config; - MockOperationEnqueuer enqueuer; - std::unique_ptr<FNetListener> fnet_listener; + MockOperationDispatcher dispatcher; + std::unique_ptr<SharedRpcResources> shared_rpc_resources; + std::unique_ptr<ClusterControllerApiRpcService> cc_service; DummyReturnHandler return_handler; bool request_is_detached{false}; FRT_RPCRequest* bound_request{nullptr}; @@ -52,13 +56,16 @@ struct FixtureBase { { config.getConfig("stor-server").set("node_index", "1"); addSlobrokConfig(config, slobrok); - fnet_listener = std::make_unique<FNetListener>(enqueuer, config.getConfigId(), 0); + + shared_rpc_resources = std::make_unique<SharedRpcResources>(config.getConfigId(), 0, 1); + cc_service = std::make_unique<ClusterControllerApiRpcService>(dispatcher, *shared_rpc_resources); + shared_rpc_resources->start_server_and_register_slobrok("my_cool_rpc_test"); } virtual ~FixtureBase() { // Must destroy any associated message contexts that may have refs to FRT_Request // instance _before_ we destroy the request itself. - enqueuer._enqueued.clear(); + dispatcher._enqueued.clear(); if (bound_request) { bound_request->SubRef(); } @@ -92,29 +99,29 @@ struct SetStateFixture : FixtureBase { void assert_enqueued_operation_has_bundle(const lib::ClusterStateBundle& expectedBundle) { ASSERT_TRUE(bound_request != nullptr); ASSERT_TRUE(request_is_detached); - ASSERT_EQ(1, enqueuer._enqueued.size()); - auto& state_request = dynamic_cast<const api::SetSystemStateCommand&>(*enqueuer._enqueued[0]); + ASSERT_EQ(1, dispatcher._enqueued.size()); + auto& state_request = dynamic_cast<const api::SetSystemStateCommand&>(*dispatcher._enqueued[0]); ASSERT_EQ(expectedBundle, state_request.getClusterStateBundle()); } void assert_request_received_and_propagated(const lib::ClusterStateBundle& bundle) { create_request(bundle); - fnet_listener->RPC_setDistributionStates(bound_request); + cc_service->RPC_setDistributionStates(bound_request); assert_enqueued_operation_has_bundle(bundle); } void assert_request_returns_error_response(RPCRequestWrapper::ErrorCode error_code) { - fnet_listener->RPC_setDistributionStates(bound_request); + cc_service->RPC_setDistributionStates(bound_request); ASSERT_FALSE(request_is_detached); ASSERT_TRUE(bound_request->IsError()); ASSERT_EQ(static_cast<uint32_t>(error_code), bound_request->GetErrorCode()); } - lib::ClusterStateBundle dummy_baseline_bundle() const { + static lib::ClusterStateBundle dummy_baseline_bundle() { return lib::ClusterStateBundle(lib::ClusterState("version:123 distributor:3 storage:3")); } - lib::ClusterStateBundle dummy_baseline_bundle_with_deferred_activation(bool deferred) const { + static lib::ClusterStateBundle dummy_baseline_bundle_with_deferred_activation(bool deferred) { return lib::ClusterStateBundle(lib::ClusterState("version:123 distributor:3 storage:3"), {}, deferred); } }; @@ -134,14 +141,14 @@ vespalib::string make_compressable_state_string() { } // anon namespace -TEST_F(FNetListenerTest, baseline_set_distribution_states_rpc_enqueues_command_with_state_bundle) { +TEST_F(ClusterControllerApiRpcServiceTest, baseline_set_distribution_states_rpc_enqueues_command_with_state_bundle) { SetStateFixture f; auto baseline = f.dummy_baseline_bundle(); f.assert_request_received_and_propagated(baseline); } -TEST_F(FNetListenerTest, set_distribution_states_rpc_with_derived_enqueues_command_with_state_bundle) { +TEST_F(ClusterControllerApiRpcServiceTest, set_distribution_states_rpc_with_derived_enqueues_command_with_state_bundle) { SetStateFixture f; lib::ClusterStateBundle spaces_bundle( lib::ClusterState("version:123 distributor:3 storage:3"), @@ -151,7 +158,7 @@ TEST_F(FNetListenerTest, set_distribution_states_rpc_with_derived_enqueues_comma f.assert_request_received_and_propagated(spaces_bundle); } -TEST_F(FNetListenerTest, compressed_bundle_is_transparently_uncompressed) { +TEST_F(ClusterControllerApiRpcServiceTest, compressed_bundle_is_transparently_uncompressed) { SetStateFixture f; auto state_str = make_compressable_state_string(); lib::ClusterStateBundle compressable_bundle{lib::ClusterState(state_str)}; @@ -160,38 +167,38 @@ TEST_F(FNetListenerTest, compressed_bundle_is_transparently_uncompressed) { // First verify that the bundle is sent in compressed form ASSERT_LT(f.bound_request->GetParams()->GetValue(2)._data._len, state_str.size()); // Ensure we uncompress it to the original form - f.fnet_listener->RPC_setDistributionStates(f.bound_request); + f.cc_service->RPC_setDistributionStates(f.bound_request); f.assert_enqueued_operation_has_bundle(compressable_bundle); } -TEST_F(FNetListenerTest, set_distribution_rpc_is_immediately_failed_if_listener_is_closed) { +TEST_F(ClusterControllerApiRpcServiceTest, set_distribution_rpc_is_immediately_failed_if_listener_is_closed) { SetStateFixture f; f.create_request(f.dummy_baseline_bundle()); - f.fnet_listener->close(); + f.cc_service->close(); f.assert_request_returns_error_response(RPCRequestWrapper::ERR_NODE_SHUTTING_DOWN); } -TEST_F(FNetListenerTest, overly_large_uncompressed_bundle_size_parameter_returns_rpc_error) { +TEST_F(ClusterControllerApiRpcServiceTest, overly_large_uncompressed_bundle_size_parameter_returns_rpc_error) { SetStateFixture f; auto encoded_bundle = f.codec.encode(f.dummy_baseline_bundle()); - f.bind_request_params(encoded_bundle, FNetListener::StateBundleMaxUncompressedSize + 1); + f.bind_request_params(encoded_bundle, ClusterControllerApiRpcService::StateBundleMaxUncompressedSize + 1); f.assert_request_returns_error_response(RPCRequestWrapper::ERR_BAD_REQUEST); } -TEST_F(FNetListenerTest, mismatching_uncompressed_bundle_size_parameter_returns_rpc_error) { +TEST_F(ClusterControllerApiRpcServiceTest, mismatching_uncompressed_bundle_size_parameter_returns_rpc_error) { SetStateFixture f; auto encoded_bundle = f.codec.encode(f.dummy_baseline_bundle()); f.bind_request_params(encoded_bundle, encoded_bundle._buffer->getDataLen() + 100); f.assert_request_returns_error_response(RPCRequestWrapper::ERR_BAD_REQUEST); } -TEST_F(FNetListenerTest, true_deferred_activation_flag_can_be_roundtrip_encoded) { +TEST_F(ClusterControllerApiRpcServiceTest, true_deferred_activation_flag_can_be_roundtrip_encoded) { SetStateFixture f; f.assert_request_received_and_propagated(f.dummy_baseline_bundle_with_deferred_activation(true)); } -TEST_F(FNetListenerTest, false_deferred_activation_flag_can_be_roundtrip_encoded) { +TEST_F(ClusterControllerApiRpcServiceTest, false_deferred_activation_flag_can_be_roundtrip_encoded) { SetStateFixture f; f.assert_request_received_and_propagated(f.dummy_baseline_bundle_with_deferred_activation(false)); } @@ -217,19 +224,19 @@ struct ActivateStateFixture : FixtureBase { void assert_enqueued_operation_has_activate_version(uint32_t version) { ASSERT_TRUE(bound_request != nullptr); ASSERT_TRUE(request_is_detached); - ASSERT_EQ(1, enqueuer._enqueued.size()); - auto& state_request = dynamic_cast<const api::ActivateClusterStateVersionCommand&>(*enqueuer._enqueued[0]); + ASSERT_EQ(1, dispatcher._enqueued.size()); + auto& state_request = dynamic_cast<const api::ActivateClusterStateVersionCommand&>(*dispatcher._enqueued[0]); ASSERT_EQ(version, state_request.version()); } void assert_request_received_and_propagated(uint32_t activate_version) { create_request(activate_version); - fnet_listener->RPC_activateClusterStateVersion(bound_request); + cc_service->RPC_activateClusterStateVersion(bound_request); assert_enqueued_operation_has_activate_version(activate_version); } }; -TEST_F(FNetListenerTest, activate_cluster_state_version_rpc_enqueues_command_with_version) { +TEST_F(ClusterControllerApiRpcServiceTest, activate_cluster_state_version_rpc_enqueues_command_with_version) { ActivateStateFixture f; f.assert_request_received_and_propagated(1234567); } diff --git a/storage/src/tests/storageserver/rpc/gtest_runner.cpp b/storage/src/tests/storageserver/rpc/gtest_runner.cpp new file mode 100644 index 00000000000..f3f903a8b46 --- /dev/null +++ b/storage/src/tests/storageserver/rpc/gtest_runner.cpp @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> + +#include <vespa/log/log.h> +LOG_SETUP("storage_storageserver_rpc_gtest_runner"); + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/storage/src/tests/storageserver/rpc/message_codec_provider_test.cpp b/storage/src/tests/storageserver/rpc/message_codec_provider_test.cpp new file mode 100644 index 00000000000..0d4e3b8df93 --- /dev/null +++ b/storage/src/tests/storageserver/rpc/message_codec_provider_test.cpp @@ -0,0 +1,46 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/storage/storageserver/rpc/message_codec_provider.h> +#include <vespa/storageapi/mbusprot/protocolserialization7.h> +#include <vespa/document/base/testdocman.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace ::testing; + +namespace storage::rpc { + +struct MessageCodecProviderTest : Test { + std::shared_ptr<const document::DocumentTypeRepo> _repo1; + std::shared_ptr<const document::DocumentTypeRepo> _repo2; + std::shared_ptr<const documentapi::LoadTypeSet> _load_types1; + std::shared_ptr<const documentapi::LoadTypeSet> _load_types2; + MessageCodecProvider _provider; + + // We don't care about repo/set contents, just their pointer identities + MessageCodecProviderTest() + : _repo1(document::TestDocRepo().getTypeRepoSp()), + _repo2(document::TestDocRepo().getTypeRepoSp()), + _load_types1(std::make_shared<documentapi::LoadTypeSet>()), + _load_types2(std::make_shared<documentapi::LoadTypeSet>()), + _provider(_repo1, _load_types1) + {} + ~MessageCodecProviderTest() override; +}; + +MessageCodecProviderTest::~MessageCodecProviderTest() = default; + +TEST_F(MessageCodecProviderTest, initially_provides_constructed_repos) { + auto wrapped = _provider.wrapped_codec(); + EXPECT_EQ(&wrapped->codec().type_repo(), _repo1.get()); + EXPECT_EQ(&wrapped->codec().load_type_set(), _load_types1.get()); +} + +TEST_F(MessageCodecProviderTest, updated_repos_reflected_in_new_wrapped_codec) { + _provider.update_atomically(_repo2, _load_types2); + + auto wrapped = _provider.wrapped_codec(); + EXPECT_EQ(&wrapped->codec().type_repo(), _repo2.get()); + EXPECT_EQ(&wrapped->codec().load_type_set(), _load_types2.get()); +} + +} diff --git a/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp new file mode 100644 index 00000000000..69259ee08ec --- /dev/null +++ b/storage/src/tests/storageserver/rpc/storage_api_rpc_service_test.cpp @@ -0,0 +1,307 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/base/testdocman.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_document_bucket.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/slobrok/sbmirror.h> +#include <vespa/storage/storageserver/rpc/storage_api_rpc_service.h> +#include <vespa/storage/storageserver/rpc/shared_rpc_resources.h> +#include <vespa/storage/storageserver/rpc/message_codec_provider.h> +#include <vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h> +#include <vespa/storage/storageserver/communicationmanager.h> +#include <vespa/storage/storageserver/rpcrequestwrapper.h> +#include <vespa/storage/storageserver/message_dispatcher.h> +#include <vespa/storageapi/message/persistence.h> +#include <tests/common/testhelper.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/host_name.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <gmock/gmock.h> +#include <condition_variable> +#include <deque> +#include <functional> +#include <memory> +#include <mutex> +#include <stdexcept> +#include <string> + +#include <thread> + +using namespace ::testing; +using namespace document::test; +using namespace std::chrono_literals; + +namespace storage::rpc { + +namespace { + +constexpr std::chrono::duration message_timeout = 60s; +constexpr std::chrono::duration slobrok_register_timeout = 60s; + +class LockingMockOperationDispatcher : public MessageDispatcher { + using MessageQueueType = std::deque<std::shared_ptr<api::StorageMessage>>; + + mutable std::mutex _mutex; + mutable std::condition_variable _cond; + MessageQueueType _enqueued; +public: + LockingMockOperationDispatcher(); + ~LockingMockOperationDispatcher() override; + + void dispatch_sync(std::shared_ptr<api::StorageMessage> msg) override { + std::lock_guard lock(_mutex); + _enqueued.emplace_back(std::move(msg)); + _cond.notify_all(); + } + + void dispatch_async(std::shared_ptr<api::StorageMessage> msg) override { + std::lock_guard lock(_mutex); + _enqueued.emplace_back(std::move(msg)); + _cond.notify_all(); + } + + [[nodiscard]] bool empty() const noexcept { + std::lock_guard lock(_mutex); + return _enqueued.empty(); + } + + void wait_until_n_messages_received(size_t n) const { + std::unique_lock lock(_mutex); + const auto deadline = std::chrono::steady_clock::now() + message_timeout; + if (!_cond.wait_until(lock, deadline, [this, n]{ return (_enqueued.size() == n); })) { + throw std::runtime_error("Timed out waiting for message"); + } + } + + [[nodiscard]] std::shared_ptr<api::StorageMessage> pop_first_message() { + std::lock_guard lock(_mutex); + assert(!_enqueued.empty()); + auto msg = std::move(_enqueued.front()); + _enqueued.pop_front(); + return msg; + } +}; + +LockingMockOperationDispatcher::LockingMockOperationDispatcher() = default; +LockingMockOperationDispatcher::~LockingMockOperationDispatcher() = default; + +api::StorageMessageAddress make_address(uint16_t node_index, bool is_distributor) { + return {"coolcluster", (is_distributor ? lib::NodeType::DISTRIBUTOR : lib::NodeType::STORAGE), node_index}; +} + +vespalib::string to_slobrok_id(const api::StorageMessageAddress& address) { + // TODO factor out slobrok ID generation code to be independent of resolver? + return CachingRpcTargetResolver::address_to_slobrok_id(address); +} + +class StorageApiNode { + vdstestlib::DirConfig _config; + std::shared_ptr<const document::DocumentTypeRepo> _doc_type_repo; + std::shared_ptr<const documentapi::LoadTypeSet> _load_type_set; + LockingMockOperationDispatcher _messages; + std::unique_ptr<MessageCodecProvider> _codec_provider; + std::unique_ptr<SharedRpcResources> _shared_rpc_resources; + std::unique_ptr<StorageApiRpcService> _service; + api::StorageMessageAddress _node_address; + vespalib::string _slobrok_id; +public: + StorageApiNode(uint16_t node_index, bool is_distributor, const mbus::Slobrok& slobrok) + : _config(getStandardConfig(true)), + _doc_type_repo(document::TestDocRepo().getTypeRepoSp()), + _load_type_set(std::make_shared<documentapi::LoadTypeSet>()), + _node_address(make_address(node_index, is_distributor)), + _slobrok_id(to_slobrok_id(_node_address)) + { + auto& cfg = _config.getConfig("stor-server"); + cfg.set("node_index", std::to_string(node_index)); + cfg.set("is_distributor", is_distributor ? "true" : "false"); + addSlobrokConfig(_config, slobrok); + + _shared_rpc_resources = std::make_unique<SharedRpcResources>(_config.getConfigId(), 0, 1); + // TODO make codec provider into interface so we can test decode-failures more easily? + _codec_provider = std::make_unique<MessageCodecProvider>(_doc_type_repo, _load_type_set); + StorageApiRpcService::Params params; + _service = std::make_unique<StorageApiRpcService>(_messages, *_shared_rpc_resources, *_codec_provider, params); + + _shared_rpc_resources->start_server_and_register_slobrok(_slobrok_id); + // Explicitly wait until we are visible in Slobrok. Just waiting for mirror readiness is not enough. + wait_until_visible_in_slobrok(_slobrok_id); + } + + void wait_until_visible_in_slobrok(vespalib::stringref id) { + const auto deadline = std::chrono::steady_clock::now() + slobrok_register_timeout; + while (_shared_rpc_resources->slobrok_mirror().lookup(id).empty()) { + if (std::chrono::steady_clock::now() > deadline) { + throw std::runtime_error("Timed out waiting for node to be visible in Slobrok"); + } + std::this_thread::sleep_for(10ms); + } + } + + const api::StorageMessageAddress& node_address() const noexcept { return _node_address; } + + std::shared_ptr<api::PutCommand> create_dummy_put_command() const { + auto doc_type = _doc_type_repo->getDocumentType("testdoctype1"); + auto doc = std::make_shared<document::Document>(*doc_type, document::DocumentId("id:foo:testdoctype1::bar")); + doc->setFieldValue(doc->getField("hstringval"), std::make_unique<document::StringFieldValue>("hello world")); + return std::make_shared<api::PutCommand>(makeDocumentBucket(document::BucketId(0)), std::move(doc), 100); + } + + void send_request_verify_not_bounced(std::shared_ptr<api::StorageCommand> req) { + if (!_messages.empty()) { + throw std::runtime_error("Node had pending messages before send"); + } + _service->send_rpc_v1_request(std::move(req)); + if (!_messages.empty()) { + throw std::runtime_error("RPC request was bounced. Most likely due to missing Slobrok mapping"); + } + } + + void send_request(std::shared_ptr<api::StorageCommand> req) { + _service->send_rpc_v1_request(std::move(req)); + } + + // TODO move StorageTransportContext away from communicationmanager.h + // TODO refactor reply handling to avoid duping detail code with CommunicationManager? + void send_response(const std::shared_ptr<api::StorageReply>& reply) { + std::unique_ptr<StorageTransportContext> context(dynamic_cast<StorageTransportContext*>( + reply->getTransportContext().release())); + assert(context); + _service->encode_rpc_v1_response(*context->_request->raw_request(), *reply); + context->_request->returnRequest(); + } + + [[nodiscard]] std::shared_ptr<api::StorageMessage> wait_and_receive_single_message() { + _messages.wait_until_n_messages_received(1); + return _messages.pop_first_message(); + } +}; + +} // anonymous namespace + +// TODO consider completely mocking Slobrok to avoid any race conditions during node registration +struct StorageApiRpcServiceTest : Test { + mbus::Slobrok _slobrok; + std::unique_ptr<StorageApiNode> _node_0; + std::unique_ptr<StorageApiNode> _node_1; + + StorageApiRpcServiceTest() + : _slobrok(), + _node_0(std::make_unique<StorageApiNode>(1, true, _slobrok)), + _node_1(std::make_unique<StorageApiNode>(4, false, _slobrok)) + { + // FIXME ugh, this isn't particularly pretty... + _node_0->wait_until_visible_in_slobrok(to_slobrok_id(_node_1->node_address())); + _node_1->wait_until_visible_in_slobrok(to_slobrok_id(_node_0->node_address())); + } + ~StorageApiRpcServiceTest() override; + + static api::StorageMessageAddress non_existing_address() { + return make_address(100, false); + } + + [[nodiscard]] std::shared_ptr<api::PutCommand> send_and_receive_put_command_at_node_1( + const std::function<void(api::PutCommand&)>& req_mutator) { + auto cmd = _node_0->create_dummy_put_command(); + cmd->setAddress(_node_1->node_address()); + req_mutator(*cmd); + _node_0->send_request_verify_not_bounced(cmd); + + auto recv_msg = _node_1->wait_and_receive_single_message(); + auto recv_as_put = std::dynamic_pointer_cast<api::PutCommand>(recv_msg); + assert(recv_as_put); + return recv_as_put; + } + [[nodiscard]] std::shared_ptr<api::PutCommand> send_and_receive_put_command_at_node_1() { + return send_and_receive_put_command_at_node_1([]([[maybe_unused]] auto& cmd){}); + } + + [[nodiscard]] std::shared_ptr<api::PutReply> respond_and_receive_put_reply_at_node_0( + const std::shared_ptr<api::PutCommand>& cmd, + const std::function<void(api::StorageReply&)>& reply_mutator) { + auto reply = std::shared_ptr<api::StorageReply>(cmd->makeReply()); + reply_mutator(*reply); + _node_1->send_response(reply); + + auto recv_reply = _node_0->wait_and_receive_single_message(); + auto recv_as_put_reply = std::dynamic_pointer_cast<api::PutReply>(recv_reply); + assert(recv_as_put_reply); + return recv_as_put_reply; + } + + [[nodiscard]] std::shared_ptr<api::PutReply> respond_and_receive_put_reply_at_node_0( + const std::shared_ptr<api::PutCommand>& cmd) { + return respond_and_receive_put_reply_at_node_0(cmd, []([[maybe_unused]] auto& reply){}); + } +}; + +StorageApiRpcServiceTest::~StorageApiRpcServiceTest() = default; + +TEST_F(StorageApiRpcServiceTest, can_send_and_respond_to_request_end_to_end) { + auto cmd = _node_0->create_dummy_put_command(); + cmd->setAddress(_node_1->node_address()); + _node_0->send_request_verify_not_bounced(cmd); + + auto recv_msg = _node_1->wait_and_receive_single_message(); + auto* put_cmd = dynamic_cast<api::PutCommand*>(recv_msg.get()); + ASSERT_TRUE(put_cmd != nullptr); + auto reply = std::shared_ptr<api::StorageReply>(put_cmd->makeReply()); + _node_1->send_response(reply); + + auto recv_reply = _node_0->wait_and_receive_single_message(); + auto* put_reply = dynamic_cast<api::PutReply*>(recv_reply.get()); + ASSERT_TRUE(put_reply != nullptr); +} + +TEST_F(StorageApiRpcServiceTest, send_to_unknown_address_bounces_with_error_reply) { + auto cmd = _node_0->create_dummy_put_command(); + cmd->setAddress(non_existing_address()); + _node_0->send_request(cmd); + + auto bounced_msg = _node_0->wait_and_receive_single_message(); + auto* put_reply = dynamic_cast<api::PutReply*>(bounced_msg.get()); + ASSERT_TRUE(put_reply != nullptr); + + auto expected_code = static_cast<api::ReturnCode::Result>(mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE); + auto expected_msg = vespalib::make_string( + "The address of service '%s' could not be resolved. It is not currently " + "registered with the Vespa name server. " + "The service must be having problems, or the routing configuration is wrong. " + "Address resolution attempted from host '%s'", + to_slobrok_id(non_existing_address()).c_str(), vespalib::HostName::get().c_str()); + + EXPECT_EQ(put_reply->getResult(), api::ReturnCode(expected_code, expected_msg)); +} + +TEST_F(StorageApiRpcServiceTest, request_metadata_is_propagated_to_receiver) { + auto recv_cmd = send_and_receive_put_command_at_node_1([](auto& cmd){ + cmd.getTrace().setLevel(7); + cmd.setTimeout(1337s); + }); + EXPECT_EQ(recv_cmd->getTrace().getLevel(), 7); + EXPECT_EQ(recv_cmd->getTimeout(), 1337s); +} + +TEST_F(StorageApiRpcServiceTest, response_trace_is_propagated_to_sender) { + auto recv_cmd = send_and_receive_put_command_at_node_1([](auto& cmd){ + cmd.getTrace().setLevel(1); + }); + auto recv_reply = respond_and_receive_put_reply_at_node_0(recv_cmd, [](auto& reply){ + reply.getTrace().trace(1, "Doing cool things", false); + }); + auto trace_str = recv_reply->getTrace().toString(); + EXPECT_THAT(trace_str, HasSubstr("Doing cool things")); +} + +TEST_F(StorageApiRpcServiceTest, response_trace_only_propagated_if_trace_level_set) { + auto recv_cmd = send_and_receive_put_command_at_node_1(); + auto recv_reply = respond_and_receive_put_reply_at_node_0(recv_cmd, [](auto& reply){ + reply.getTrace().trace(1, "Doing cool things", false); + }); + auto trace_str = recv_reply->getTrace().toString(); + EXPECT_THAT(trace_str, Not(HasSubstr("Doing cool things"))); +} + +} diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 3e4b1fd6515..3babc98cbb1 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -55,3 +55,19 @@ mbus.skip_request_thread bool default=false ## Skip communication manager thread on mbus requests ## Experimental skip_thread bool default=false + +## Whether to use direct P2P RPC protocol for all StorageAPI communication +## instead of going via MessageBus. +use_direct_storageapi_rpc bool default=false + +## The number of network (FNET) threads used by the shared rpc resource. +rpc.num_network_threads int default=1 + +# Minimum size of packets to compress (0 means no compression) +rpc.compress.limit int default=1024 + +## Compression level for packets +rpc.compress.level int default=3 + +## Compression type for packets. +rpc.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 606d61ab944..538bff5b9f0 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + vespa_add_library(storage_storageserver SOURCES bouncer.cpp @@ -12,7 +13,6 @@ vespa_add_library(storage_storageserver distributornodecontext.cpp documentapiconverter.cpp fnet_metrics_wrapper.cpp - fnetlistener.cpp mergethrottler.cpp messagesink.cpp opslogger.cpp @@ -21,13 +21,13 @@ vespa_add_library(storage_storageserver service_layer_error_listener.cpp servicelayernode.cpp servicelayernodecontext.cpp - slime_cluster_state_bundle_codec.cpp statemanager.cpp statereporter.cpp storagemetricsset.cpp storagenode.cpp storagenodecontext.cpp tls_statistics_metrics_wrapper.cpp + $<TARGET_OBJECTS:storage_storageserver_rpc> INSTALL lib64 DEPENDS storage diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index b51394e2e64..5471d66a864 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "communicationmanager.h" -#include "fnetlistener.h" #include "rpcrequestwrapper.h" #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> #include <vespa/messagebus/emptyreply.h> @@ -11,6 +10,10 @@ #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/storageserver/configurable_bucket_resolver.h> +#include <vespa/storage/storageserver/rpc/shared_rpc_resources.h> +#include <vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h> +#include <vespa/storage/storageserver/rpc/message_codec_provider.h> +#include <vespa/storage/storageserver/rpc/storage_api_rpc_service.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -20,6 +23,7 @@ #include <vespa/log/bufferedlogger.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/documentapi/messagebus/messages/getdocumentreply.h> +#include <string_view> LOG_SETUP(".communication.manager"); @@ -242,20 +246,32 @@ struct PlaceHolderBucketResolver : public BucketResolver { } }; +vespalib::compression::CompressionConfig +convert_to_rpc_compression_config(const vespa::config::content::core::StorCommunicationmanagerConfig& mgr_config) { + using vespalib::compression::CompressionConfig; + using vespa::config::content::core::StorCommunicationmanagerConfig; + auto compression_type = CompressionConfig::toType( + StorCommunicationmanagerConfig::Rpc::Compress::getTypeName(mgr_config.rpc.compress.type).c_str()); + return CompressionConfig(compression_type, mgr_config.rpc.compress.level, 90, mgr_config.rpc.compress.limit); +} + } CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri) : StorageLink("Communication manager"), _component(compReg, "communicationmanager"), _metrics(_component.getLoadTypes()->getMetricLoadTypes()), - _listener(), + _shared_rpc_resources(), // Created upon initial configuration + _storage_api_rpc_service(), // (ditto) + _cc_rpc_service(), // (ditto) _eventQueue(), _mbus(), _configUri(configUri), _closed(false), _docApiConverter(configUri, std::make_shared<PlaceHolderBucketResolver>()), _thread(), - _skip_thread(false) + _skip_thread(false), + _use_direct_storageapi_rpc(false) { _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); _component.registerMetric(_metrics); @@ -270,8 +286,8 @@ CommunicationManager::onOpen() framework::MilliSecTime maxProcessingTime(60 * 1000); _thread = _component.startThread(*this, maxProcessingTime); - if (_listener) { - _listener->registerHandle(_component.getIdentity()); + if (_shared_rpc_resources) { + _shared_rpc_resources->start_server_and_register_slobrok(_component.getIdentity()); } } @@ -313,8 +329,13 @@ void CommunicationManager::onClose() } } - if (_listener) { - _listener->close(); + // TODO remove? this no longer has any particularly useful semantics + if (_cc_rpc_service) { + _cc_rpc_service->close(); + } + // TODO do this after we drain queues? + if (_shared_rpc_resources) { + _shared_rpc_resources->shutdown(); } // Stopping pumper thread should stop all incoming messages from being @@ -327,6 +348,7 @@ void CommunicationManager::onClose() } // Emptying remaining queued messages + // FIXME but RPC/mbus is already shut down at this point...! Make sure we handle this std::shared_ptr<api::StorageMessage> msg; api::ReturnCode code(api::ReturnCode::ABORTED, "Node shutting down"); while (_eventQueue.size() > 0) { @@ -362,9 +384,9 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> LOG(warning, "%s", m.c_str()); _component.requestShutdown(m); } - if (_listener->getListenPort() != config->rpcport) { + if (_shared_rpc_resources->listen_port() != config->rpcport) { auto m = make_string("rpc port changed from %d to %d. Will conduct a quick, but controlled restart.", - _listener->getListenPort(), config->rpcport); + _shared_rpc_resources->listen_port(), config->rpcport); LOG(warning, "%s", m.c_str()); _component.requestShutdown(m); } @@ -407,7 +429,15 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> configureMessageBusLimits(*config); } - _listener = std::make_unique<FNetListener>(*this, _configUri, config->rpcport); + _use_direct_storageapi_rpc = config->useDirectStorageapiRpc; + _message_codec_provider = std::make_unique<rpc::MessageCodecProvider>(_component.getTypeRepo()->documentTypeRepo, + _component.getLoadTypes()); + _shared_rpc_resources = std::make_unique<rpc::SharedRpcResources>(_configUri, config->rpcport, config->rpc.numNetworkThreads); + _cc_rpc_service = std::make_unique<rpc::ClusterControllerApiRpcService>(*this, *_shared_rpc_resources); + rpc::StorageApiRpcService::Params rpc_params; + rpc_params.compression_config = convert_to_rpc_compression_config(*config); + _storage_api_rpc_service = std::make_unique<rpc::StorageApiRpcService>( + *this, *_shared_rpc_resources, *_message_codec_provider, rpc_params); if (_mbus) { mbus::DestinationSessionParams dstParams; @@ -452,15 +482,17 @@ CommunicationManager::enqueue_or_process(std::shared_ptr<api::StorageMessage> ms LOG(spam, "Process storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); process(msg); } else { - enqueue(std::move(msg)); + dispatch_async(std::move(msg)); } } -void -CommunicationManager::enqueue(std::shared_ptr<api::StorageMessage> msg) -{ - assert(msg); - LOG(spam, "Enq storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); +void CommunicationManager::dispatch_sync(std::shared_ptr<api::StorageMessage> msg) { + LOG(spam, "Direct dispatch of storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); + process(std::move(msg)); +} + +void CommunicationManager::dispatch_async(std::shared_ptr<api::StorageMessage> msg) { + LOG(spam, "Enqueued dispatch of storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); _eventQueue.enqueue(std::move(msg)); } @@ -542,15 +574,18 @@ CommunicationManager::sendCommand( switch (address.getProtocol()) { case api::StorageMessageAddress::STORAGE: { - LOG(spam, "Send to %s: %s", address.toString().c_str(), msg->toString().c_str()); - - auto cmd = std::make_unique<mbusprot::StorageCommand>(msg); + LOG(debug, "Send to %s: %s", address.toString().c_str(), msg->toString().c_str()); + if (_use_direct_storageapi_rpc) { + _storage_api_rpc_service->send_rpc_v1_request(msg); + } else { + auto cmd = std::make_unique<mbusprot::StorageCommand>(msg); - cmd->setContext(mbus::Context(msg->getMsgId())); - cmd->setRetryEnabled(address.retryEnabled()); - cmd->setTimeRemaining(msg->getTimeout()); - cmd->setTrace(msg->getTrace()); - sendMessageBusMessage(msg, std::move(cmd), address.getRoute()); + cmd->setContext(mbus::Context(msg->getMsgId())); + cmd->setRetryEnabled(address.retryEnabled()); + cmd->setTimeRemaining(msg->getTimeout()); + cmd->setTrace(msg->getTrace()); + sendMessageBusMessage(msg, std::move(cmd), address.getRoute()); + } break; } case api::StorageMessageAddress::DOCUMENT: @@ -601,8 +636,11 @@ CommunicationManager::sendDirectRPCReply( RPCRequestWrapper& request, const std::shared_ptr<api::StorageReply>& reply) { - std::string requestName(request.getMethodName()); - if (requestName == "getnodestate3") { + std::string_view requestName(request.getMethodName()); // TODO non-name based dispatch + // TODO rework this entire dispatch mechanism :D + if (requestName == "storageapi.v1.send") { + _storage_api_rpc_service->encode_rpc_v1_response(*request.raw_request(), *reply); + } else if (requestName == "getnodestate3") { auto& gns(dynamic_cast<api::GetNodeStateReply&>(*reply)); std::ostringstream ns; serializeNodeState(gns, ns, true, true, false); @@ -752,7 +790,7 @@ CommunicationManager::print(std::ostream& out, bool verbose, const std::string& } void CommunicationManager::updateMessagebusProtocol( - const std::shared_ptr<const document::DocumentTypeRepo> &repo) { + const std::shared_ptr<const document::DocumentTypeRepo>& repo) { if (_mbus) { framework::SecondTime now(_component.getClock().getTimeInSeconds()); auto newDocumentProtocol = std::make_shared<documentapi::DocumentProtocol>(*_component.getLoadTypes(), repo); @@ -761,6 +799,9 @@ void CommunicationManager::updateMessagebusProtocol( auto newStorageProtocol = std::make_shared<mbusprot::StorageProtocol>(repo, *_component.getLoadTypes()); _earlierGenerations.push_back(std::make_pair(now, _mbus->getMessageBus().putProtocol(newStorageProtocol))); } + if (_message_codec_provider) { + _message_codec_provider->update_atomically(repo, _component.getLoadTypes()); + } } void CommunicationManager::updateBucketSpacesConfig(const BucketspacesConfig& config) { diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 23b59f5a42a..7ac9d575ee6 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -12,7 +12,7 @@ #include "communicationmanagermetrics.h" #include "documentapiconverter.h" -#include "message_enqueuer.h" +#include "message_dispatcher.h" #include <vespa/storage/common/storagelink.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/config/config-stor-communicationmanager.h> @@ -37,19 +37,24 @@ namespace mbus { } namespace storage { +namespace rpc { +class ClusterControllerApiRpcService; +class MessageCodecProvider; +class SharedRpcResources; +class StorageApiRpcService; +} + struct BucketResolver; -class VisitorMbusSession; class Visitor; class VisitorThread; -class FNetListener; class RPCRequestWrapper; class StorageTransportContext : public api::TransportContext { public: - StorageTransportContext(std::unique_ptr<documentapi::DocumentMessage> msg); - StorageTransportContext(std::unique_ptr<mbusprot::StorageCommand> msg); - StorageTransportContext(std::unique_ptr<RPCRequestWrapper> request); - ~StorageTransportContext(); + explicit StorageTransportContext(std::unique_ptr<documentapi::DocumentMessage> msg); + explicit StorageTransportContext(std::unique_ptr<mbusprot::StorageCommand> msg); + explicit StorageTransportContext(std::unique_ptr<RPCRequestWrapper> request); + ~StorageTransportContext() override; std::unique_ptr<documentapi::DocumentMessage> _docAPIMsg; std::unique_ptr<mbusprot::StorageCommand> _storageProtocolMsg; @@ -63,7 +68,7 @@ class CommunicationManager final public mbus::IMessageHandler, public mbus::IReplyHandler, private framework::MetricUpdateHook, - public MessageEnqueuer + public MessageDispatcher { private: CommunicationManager(const CommunicationManager&); @@ -72,7 +77,10 @@ private: StorageComponent _component; CommunicationManagerMetrics _metrics; - std::unique_ptr<FNetListener> _listener; + std::unique_ptr<rpc::SharedRpcResources> _shared_rpc_resources; + std::unique_ptr<rpc::StorageApiRpcService> _storage_api_rpc_service; + std::unique_ptr<rpc::ClusterControllerApiRpcService> _cc_rpc_service; + std::unique_ptr<rpc::MessageCodecProvider> _message_codec_provider; Queue _eventQueue; // XXX: Should perhaps use a configsubscriber and poll from StorageComponent ? std::unique_ptr<config::ConfigFetcher> _configFetcher; @@ -112,6 +120,7 @@ private: DocumentApiConverter _docApiConverter; framework::Thread::UP _thread; bool _skip_thread; + bool _use_direct_storageapi_rpc; void updateMetrics(const MetricLockGuard &) override; void enqueue_or_process(std::shared_ptr<api::StorageMessage> msg); @@ -122,9 +131,12 @@ private: public: CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri); - ~CommunicationManager(); + ~CommunicationManager() override; + + // MessageDispatcher overrides + void dispatch_sync(std::shared_ptr<api::StorageMessage> msg) override; + void dispatch_async(std::shared_ptr<api::StorageMessage> msg) override; - void enqueue(std::shared_ptr<api::StorageMessage> msg) override; mbus::RPCMessageBus& getMessageBus() { assert(_mbus.get()); return *_mbus; } const PriorityConverter& getPriorityConverter() const { return _docApiConverter.getPriorityConverter(); } diff --git a/storage/src/vespa/storage/storageserver/fnetlistener.h b/storage/src/vespa/storage/storageserver/fnetlistener.h deleted file mode 100644 index e37727beb44..00000000000 --- a/storage/src/vespa/storage/storageserver/fnetlistener.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/slobrok/sbregister.h> -#include <atomic> - -class FNET_Transport; -class FastOS_ThreadPool; - -namespace storage { - -namespace api { class StorageMessage; } - -class MessageEnqueuer; - -class FNetListener : public FRT_Invokable -{ -public: - static constexpr uint32_t StateBundleMaxUncompressedSize = 1024 * 1024 * 16; - - FNetListener(MessageEnqueuer& messageEnqueuer, - const config::ConfigUri & configUri, - uint32_t port); - ~FNetListener() override; - - void initRPC(); - void RPC_getNodeState2(FRT_RPCRequest *req); - void RPC_setSystemState2(FRT_RPCRequest *req); - void RPC_getCurrentTime(FRT_RPCRequest *req); - void RPC_setDistributionStates(FRT_RPCRequest* req); - void RPC_activateClusterStateVersion(FRT_RPCRequest* req); - - void registerHandle(vespalib::stringref handle); - void close(); - int getListenPort() const; - -private: - MessageEnqueuer& _messageEnqueuer; - std::unique_ptr<FastOS_ThreadPool> _threadPool; - std::unique_ptr<FNET_Transport> _transport; - std::unique_ptr<FRT_Supervisor> _orb; - std::atomic<bool> _closed; - slobrok::api::RegisterAPI _slobrokRegister; - vespalib::string _handle; - - void detach_and_forward_to_enqueuer(std::shared_ptr<api::StorageMessage> cmd, FRT_RPCRequest *req); -}; - -} diff --git a/storage/src/vespa/storage/storageserver/message_dispatcher.h b/storage/src/vespa/storage/storageserver/message_dispatcher.h new file mode 100644 index 00000000000..e483d55f753 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/message_dispatcher.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> + +namespace storage { + +namespace api { class StorageMessage; } + +/** + * Allows for dispatching messages either as a sync or async operation. + * Semantics: + * - dispatch_sync: no immediate thread handoff; try to process in caller thread if possible + * - dispatch_async: guaranteed thread handoff; message not processed in caller thread. + */ +class MessageDispatcher { +public: + virtual ~MessageDispatcher() = default; + + virtual void dispatch_sync(std::shared_ptr<api::StorageMessage> msg) = 0; + virtual void dispatch_async(std::shared_ptr<api::StorageMessage> msg) = 0; +}; + +} diff --git a/storage/src/vespa/storage/storageserver/message_enqueuer.h b/storage/src/vespa/storage/storageserver/message_enqueuer.h deleted file mode 100644 index 921328a054b..00000000000 --- a/storage/src/vespa/storage/storageserver/message_enqueuer.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> - -namespace storage { -namespace api { class StorageMessage; } - -// Interface for enqueuing a StorageMessage for further processing -class MessageEnqueuer { -public: - virtual ~MessageEnqueuer() = default; - virtual void enqueue(std::shared_ptr<api::StorageMessage> msg) = 0; -}; - -} diff --git a/storage/src/vespa/storage/storageserver/messagedispatcher.cpp b/storage/src/vespa/storage/storageserver/messagedispatcher.cpp deleted file mode 100644 index e6b66c7065c..00000000000 --- a/storage/src/vespa/storage/storageserver/messagedispatcher.cpp +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messagedispatcher.h" - -#include <vespa/storageapi/message/state.h> -#include <storageapi/messageapi/chainedcommand.h> -#include <storageapi/messageapi/chainedreply.h> -#include <vespa/document/bucket/bucketid.h> - -#include <vespa/log/log.h> -LOG_SETUP(".message.dispatcher"); - -using std::shared_ptr; - -namespace storage { - -MessageDispatcher::MessageDispatcher(StorageServerInterface& server) - : StorageLink(), - _access(), - _cache(), - _systemState(""), - _server(server) -{ -} - -MessageDispatcher::~MessageDispatcher() -{ - closeNextLink(); - LOG(debug, "Deleting link %s.", toString().c_str()); -} - -void -MessageDispatcher::onClose() -{ - vespalib::LockGuard lock(_access); - for (std::map<api::StorageMessage::Id, std::shared_ptr<ReplyPair> > - ::iterator it = _cache.begin(); it != _cache.end(); ++it) - { - std::shared_ptr<api::ChainedReply> reply(it->second->first); - if (it->second->second != 0) { - reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, - "Storage node closing down. Aborting command.")); - sendUp(reply); - it->second->second = 0; - } - } - -} - -void -MessageDispatcher::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "MessageDispatcher()"; -} - -bool MessageDispatcher::onDown(const shared_ptr<api::StorageMessage> & msg) -{ - if (msg->getType().isReply()) { - shared_ptr<api::ChainedReply> reply( - std::dynamic_pointer_cast<api::ChainedReply>(msg)); - if (reply.get()) { - return handleReply(reply, false); - } - } else { - shared_ptr<api::ChainedCommand> cmd( - std::dynamic_pointer_cast<api::ChainedCommand>(msg)); - if (cmd.get()) { - return handleCommand(cmd); - } - if (msg->getType() == api::MessageType::SETSYSTEMSTATE) { - shared_ptr<api::SetSystemStateCommand> stateCmd( - std::dynamic_pointer_cast<api::SetSystemStateCommand>( - msg)); - assert(stateCmd.get()); - _systemState = stateCmd->getSystemState(); - LOG(debug, "Got new distributor state %s.", - _systemState.toString().c_str()); - } - } - return false; -} - -bool MessageDispatcher::onUp(const std::shared_ptr<api::StorageMessage> & msg) -{ - if (msg->getType().isReply()) { - shared_ptr<api::ChainedReply> reply( - std::dynamic_pointer_cast<api::ChainedReply>(msg)); - if (reply.get()) { - return handleReply(reply, true); - } - } - return false; -} - -bool MessageDispatcher:: -handleCommand(const std::shared_ptr<api::ChainedCommand> & cmd) -{ - // If we're the first node in the chain, - // the message has a bucket id related to it, - // and message came from wrong distributor, fail the message. - uint16_t expectedNode = 0xFFFF; - if (cmd->getSourceIndex() != 0xFFFF && - cmd->hasBucketId() && - !isCorrectDistributor(cmd->getBucketId(), cmd->getSourceIndex(), - expectedNode)) - { - std::string msg; - - if (expectedNode != 0xFFFF) { - msg = vespalib::make_string( - "Got chained command %s with bucket id %s from distributor " - "%d, which is wrong given our state. Correct should be %d. " - "Ignoring since we're primary node.", - cmd->getType().getName().c_str(), - cmd->getBucketId().toString().c_str(), - cmd->getSourceIndex(), - expectedNode); - } else { - msg = vespalib::make_string( - "Got chained command %s with bucket id %s, but no " - "distributors in system state. Haven't received system " - "state yet?", - cmd->getType().getName().c_str(), - cmd->getBucketId().toString().c_str()); - } - - LOG(debug, msg.c_str()); - shared_ptr<api::StorageReply> reply(cmd->makeReply().release()); - reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, msg)); - sendUp(reply); - return true; - - } - // If not used chained, just pass it through - if (!cmd->hasNodes()) { - LOG(spam, "Chained command contains no nodes, passing it through"); - return false; - } - bool runLocally = cmd->getNodes().back()._run; - // If last node in chain, handle directly - if (cmd->getNodeCount() == 1) { - if (runLocally) { - LOG(spam, "Last node in chain, running it locally."); - return false; - } else { - LOG(spam, "Last node in chain, not running locally, so returning."); - shared_ptr<api::StorageReply> reply(cmd->makeReply().release()); - sendUp(reply); - return true; - } - } - // Create commands first, as we need ids for cache. - shared_ptr<api::ChainedCommand> extCmd(cmd->clone()); - shared_ptr<api::ChainedCommand> localCmd(runLocally ? cmd->clone() : 0); - - // When stuff in cache, to be sure it's there when reply comes. - shared_ptr<api::ChainedReply> reply(dynamic_cast<api::ChainedReply*>( - cmd->makeReply().release())); - assert(reply.get()); - { - vespalib::LockGuard lock(_access); - shared_ptr<ReplyPair> pair(new ReplyPair(reply, runLocally ? 2 : 1)); - _cache[extCmd->getMsgId()] = pair; - if (localCmd.get()) { - _cache[localCmd->getMsgId()] = pair; - } - } - // Send external first since it will probably use the most time - extCmd->setSourceIndex(0xFFFF); - extCmd->getNodes().pop_back(); - extCmd->setAddress(api::ServerAddress(_server.getClusterName(), "storage", extCmd->getNodes().back()._node)); - - LOG(spam, "Sending chained command on to node %d.", - extCmd->getNodes().back()._node); - sendUp(extCmd); - // Send internal copy if run locally flag is set - if (runLocally) { - LOG(spam, "Running chained command locally."); - localCmd->setSourceIndex(0xFFFF); - sendDown(localCmd); - } - return true; -} - -bool -MessageDispatcher::handleReply( - const std::shared_ptr<api::ChainedReply>& reply, bool localSource) -{ - // Ignore replies on their way up in the storage chain, with a - // destination object set. These are replies on commands not sent - // locally, thus not replies possibly for the message dispatcher. - if (localSource && !reply->isLocal()) return false; - - vespalib::LockGuard lock(_access); - std::map<api::StorageMessage::Id, shared_ptr<ReplyPair> >::iterator it - = _cache.find(reply->getMsgId()); - if (it == _cache.end()) { - return false; // Not for us - } - if (it->second.get() == 0) { - LOG(debug, "Reply already sent back (probably due to shutdown)"); - return true; // Already sent - } - bool lastReply = (--it->second->second == 0); - if (!lastReply || localSource) { - it->second->first->appendState(*reply); - } else { - it->second->first->prependState(*reply); - } - if (lastReply) { - LOG(spam, "Last chained reply retrieved, sending original reply."); - sendUp(it->second->first); - } else { - LOG(spam, "Got chained reply, waiting for next"); - } - _cache.erase(it); - return true; -} - -bool -MessageDispatcher::isCorrectDistributor( - const document::BucketId& id, uint16_t distributor, uint16_t& expected) -{ - std::vector<uint16_t> distributors; - (id).getIdealNodes(lib::NodeType::DISTRIBUTOR, _systemState, _server.getBucketIdFactory(), distributors); - return (distributors.size() > 0 && (expected = distributors[0]) == distributor); -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/messagedispatcher.h b/storage/src/vespa/storage/storageserver/messagedispatcher.h deleted file mode 100644 index 0c8fdb8916b..00000000000 --- a/storage/src/vespa/storage/storageserver/messagedispatcher.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::MessageDispatcher - * @ingroup storageserver - * - * @brief Sends messages through to multiple hosts. - * - * In VDS, some messages are sent to the first storage node, and the node itself - * should send the request on to another storage node and so on (put/remove). - * This link is responsible for receiving such messages, and send it through to - * next host, as well as through to the local host, wait for both responses and - * reply back. If one of the responses fails, it should issue a revert command. - * - * @author H�kon Humberset - * @date 2006-01-16 - * @version $Id$ - */ - -#pragma once - -#include <vespa/vespalib/util/sync.h> -#include <map> -#include <vdslib/state/systemstate.h> -#include <vespa/storage/common/storagelink.h> - -namespace storage { -namespace api { - class BucketId; - class ChainedCommand; - class ChainedReply; -} - -class MessageDispatcher : public StorageLink { - mutable vespalib::Lock _access; - typedef std::pair<std::shared_ptr<api::ChainedReply>, uint32_t> ReplyPair; - std::map<api::StorageMessage::Id, std::shared_ptr<ReplyPair> > _cache; - lib::ClusterState _systemState; - StorageServerInterface& _server; - -public: - explicit MessageDispatcher(StorageServerInterface& server); - ~MessageDispatcher(); - - virtual void onClose(); - - virtual void print(std::ostream& out, bool verbose, - const std::string& indent) const; - - class Factory : public StorageLink::Factory { - public: - std::unique_ptr<StorageLink> create(const std::string& configId, - StorageServerInterface& server) const - { - (void) configId; - return std::unique_ptr<StorageLink>(new MessageDispatcher(server)); - } - }; - -private: - - bool onDown(const std::shared_ptr<api::StorageMessage> & msg); - bool onUp(const std::shared_ptr<api::StorageMessage> & msg); - - bool handleCommand(const std::shared_ptr<api::ChainedCommand>& cmd); - bool handleReply(const std::shared_ptr<api::ChainedReply>& reply, - bool localSource); - - bool isCorrectDistributor(const document::BucketId& id, uint16_t distributor, - uint16_t& expected); - -}; - -} // storage - - diff --git a/storage/src/vespa/storage/storageserver/rpc/.gitignore b/storage/src/vespa/storage/storageserver/rpc/.gitignore new file mode 100644 index 00000000000..d3594ec97d6 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/.gitignore @@ -0,0 +1,2 @@ +*.pb.h +*.pb.cc diff --git a/storage/src/vespa/storage/storageserver/rpc/CMakeLists.txt b/storage/src/vespa/storage/storageserver/rpc/CMakeLists.txt new file mode 100644 index 00000000000..21498d66781 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/CMakeLists.txt @@ -0,0 +1,25 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +find_package(Protobuf REQUIRED) +PROTOBUF_GENERATE_CPP(storage_storageserver_rpc_PROTOBUF_SRCS storage_storageserver_rpc_PROTOBUF_HDRS + protobuf/rpc_envelope.proto +) + +vespa_add_source_target(protobufgen_storage_storageserver_rpc DEPENDS + ${storage_storageserver_rpc_PROTOBUF_SRCS} + ${storage_storageserver_rpc_PROTOBUF_HDRS}) + +vespa_suppress_warnings_for_protobuf_sources(SOURCES ${storage_storageserver_rpc_PROTOBUF_SRCS}) + +vespa_add_library(storage_storageserver_rpc OBJECT + SOURCES + caching_rpc_target_resolver.cpp + cluster_controller_api_rpc_service.cpp + message_codec_provider.cpp + rpc_target.cpp + shared_rpc_resources.cpp + slime_cluster_state_bundle_codec.cpp + storage_api_rpc_service.cpp + ${storage_storageserver_rpc_PROTOBUF_SRCS} + DEPENDS +) diff --git a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp new file mode 100644 index 00000000000..5241ec6f769 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.cpp @@ -0,0 +1,104 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "caching_rpc_target_resolver.h" +#include "shared_rpc_resources.h" +#include <vespa/fnet/frt/target.h> +#include <vespa/slobrok/imirrorapi.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <cassert> + +#include <vespa/log/log.h> +LOG_SETUP(".storage.caching_rpc_target_resolver"); + +namespace storage::rpc { + +CachingRpcTargetResolver::CachingRpcTargetResolver(const slobrok::api::IMirrorAPI& slobrok_mirror, + const RpcTargetFactory& target_factory) + : _slobrok_mirror(slobrok_mirror), + _target_factory(target_factory), + _targets_rwmutex() +{ +} + +CachingRpcTargetResolver::~CachingRpcTargetResolver() = default; + +vespalib::string +CachingRpcTargetResolver::address_to_slobrok_id(const api::StorageMessageAddress& address) { + vespalib::asciistream as; + as << "storage/cluster." << address.getCluster() + << '/' << ((address.getNodeType() == lib::NodeType::STORAGE) ? "storage" : "distributor") + << '/' << address.getIndex(); + return as.str(); +} + +std::shared_ptr<RpcTarget> +CachingRpcTargetResolver::lookup_target(const vespalib::string& slobrok_id, uint32_t curr_slobrok_gen) { + std::shared_lock lock(_targets_rwmutex); + auto itr = _targets.find(slobrok_id); + if ((itr != _targets.end()) + && itr->second->_target->is_valid() + && (itr->second->_slobrok_gen == curr_slobrok_gen)) { + return itr->second; + } + return {}; +} + +std::shared_ptr<RpcTarget> +CachingRpcTargetResolver::consider_update_target(const vespalib::string& slobrok_id, + const vespalib::string& connection_spec, + uint32_t curr_slobrok_gen, + [[maybe_unused]] const UniqueLock& targets_lock) { + // If address has the same spec as the existing target, just reuse it. + auto itr = _targets.find(slobrok_id); + if ((itr != _targets.end()) + && (itr->second->_target->is_valid()) + && (itr->second->_spec == connection_spec)) + { + LOG(debug, "Updating existing mapping '%s' -> '%s' (gen %u) to gen %u", + slobrok_id.c_str(), connection_spec.c_str(), itr->second->_slobrok_gen, curr_slobrok_gen); + itr->second->_slobrok_gen = curr_slobrok_gen; + return itr->second; + } + return {}; +} + +std::shared_ptr<RpcTarget> +CachingRpcTargetResolver::insert_new_target_mapping(const vespalib::string& slobrok_id, + const vespalib::string& connection_spec, + uint32_t curr_slobrok_gen, + [[maybe_unused]] const UniqueLock& targets_lock) { + auto target = _target_factory.make_target(connection_spec, curr_slobrok_gen); // TODO expensive inside lock? + assert(target); + std::shared_ptr<RpcTarget> rpc_target(std::move(target)); + _targets[slobrok_id] = rpc_target; + LOG(debug, "Added mapping '%s' -> '%s' at gen %u", slobrok_id.c_str(), connection_spec.c_str(), curr_slobrok_gen); + return rpc_target; +} + +std::shared_ptr<RpcTarget> +CachingRpcTargetResolver::resolve_rpc_target(const api::StorageMessageAddress& address) { + // TODO or map directly from address to target instead of going via stringification? Needs hashing, if so. + auto slobrok_id = address_to_slobrok_id(address); + const uint32_t curr_slobrok_gen = _slobrok_mirror.updates(); + if (auto result = lookup_target(slobrok_id, curr_slobrok_gen)) { + return result; + } + auto specs = _slobrok_mirror.lookup(slobrok_id); // FIXME string type mismatch; implicit conv! + if (specs.empty()) { + LOG(debug, "Found no mapping for '%s'", slobrok_id.c_str()); + // TODO return potentially stale existing target if no longer existing in SB? + // TODO or clear any existing mapping? + return {}; + } + // Note: We don't use wildcards so there is a 1-to-1 mapping between service name / slobrok id and connection spec. + assert(specs.size() == 1); + const auto& connection_spec = specs[0].second; + std::unique_lock lock(_targets_rwmutex); + if (auto result = consider_update_target(slobrok_id, connection_spec, curr_slobrok_gen, lock)) { + return result; + } + return insert_new_target_mapping(slobrok_id, connection_spec, curr_slobrok_gen, lock); +} + +} diff --git a/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h new file mode 100644 index 00000000000..cf94f7545bc --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/caching_rpc_target_resolver.h @@ -0,0 +1,53 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "rpc_target.h" +#include "rpc_target_factory.h" +#include <vespa/vespalib/stllike/hash_map.h> +#include <memory> +#include <shared_mutex> + +namespace slobrok::api { class IMirrorAPI; } + +namespace storage { + +namespace api { class StorageMessageAddress; } + +namespace rpc { + +/** + * Class that resolves and caches rpc targets based on StorageMessageAddress that is mapped to slobrok id, + * with lookup in a slobrok mirror. + */ +class CachingRpcTargetResolver { +private: + const slobrok::api::IMirrorAPI& _slobrok_mirror; + const RpcTargetFactory& _target_factory; + using UniqueLock = std::unique_lock<std::shared_mutex>; + mutable std::shared_mutex _targets_rwmutex; + // TODO LRU? Size cap? + vespalib::hash_map<vespalib::string, std::shared_ptr<RpcTarget>> _targets; + + std::shared_ptr<RpcTarget> lookup_target(const vespalib::string& slobrok_id, uint32_t curr_slobrok_gen); + std::shared_ptr<RpcTarget> consider_update_target(const vespalib::string& slobrok_id, + const vespalib::string& connection_spec, + uint32_t curr_slobrok_gen, + const UniqueLock& targets_lock); + + std::shared_ptr<RpcTarget> insert_new_target_mapping(const vespalib::string& slobrok_id, + const vespalib::string& connection_spec, + uint32_t curr_slobrok_gen, + const UniqueLock& targets_lock); + +public: + CachingRpcTargetResolver(const slobrok::api::IMirrorAPI& slobrok_mirror, + const RpcTargetFactory& target_factory); + ~CachingRpcTargetResolver(); + + static vespalib::string address_to_slobrok_id(const api::StorageMessageAddress& address); + + std::shared_ptr<RpcTarget> resolve_rpc_target(const api::StorageMessageAddress& address); +}; + +} // rpc +} // storage diff --git a/storage/src/vespa/storage/storageserver/fnetlistener.cpp b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp index c5d7880d966..e34030f9bd7 100644 --- a/storage/src/vespa/storage/storageserver/fnetlistener.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.cpp @@ -1,119 +1,81 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "fnetlistener.h" -#include "communicationmanager.h" -#include "rpcrequestwrapper.h" +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "cluster_controller_api_rpc_service.h" +#include "shared_rpc_resources.h" #include "slime_cluster_state_bundle_codec.h" +#include <vespa/storage/storageserver/communicationmanager.h> +#include <vespa/storage/storageserver/message_dispatcher.h> +#include <vespa/storage/storageserver/rpcrequestwrapper.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/rpcrequest.h> #include <vespa/storageapi/message/state.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/host_name.h> -#include <vespa/vespalib/util/time.h> -#include <vespa/fnet/frt/supervisor.h> -#include <vespa/fnet/transport.h> -#include <sstream> -#include <thread> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> -LOG_SETUP(".rpc.listener"); - -namespace storage { - -FNetListener::FNetListener(MessageEnqueuer& messageEnqueuer, const config::ConfigUri & configUri, uint32_t port) - : _messageEnqueuer(messageEnqueuer), - _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)), - _transport(std::make_unique<FNET_Transport>()), - _orb(std::make_unique<FRT_Supervisor>(_transport.get())), - _closed(false), - _slobrokRegister(*_orb, configUri) -{ - initRPC(); - if (!_orb->Listen(port)) { - std::ostringstream ost; - ost << "Failed to listen to RPC port " << port << "."; - throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); - } - _transport->Start(_threadPool.get()); -} +LOG_SETUP(".storage.cluster_controller_api_rpc_service"); -FNetListener::~FNetListener() -{ - if (!_closed) { - close(); - } -} +namespace storage::rpc { -int -FNetListener::getListenPort() const +ClusterControllerApiRpcService::ClusterControllerApiRpcService( + MessageDispatcher& message_dispatcher, + SharedRpcResources& rpc_resources) + : _message_dispatcher(message_dispatcher), + _rpc_resources(rpc_resources), + _closed(false) { - return _orb->GetListenPort(); + register_server_methods(rpc_resources); } -void -FNetListener::registerHandle(vespalib::stringref handle) { - _slobrokRegister.registerName(handle); - while (_slobrokRegister.busy()) { - LOG(debug, "Waiting to register in slobrok"); - std::this_thread::sleep_for(50ms); - } - _handle = handle; -} +ClusterControllerApiRpcService::~ClusterControllerApiRpcService() = default; -void -FNetListener::close() -{ - _closed = true; - _slobrokRegister.unregisterName(_handle); - _transport->ShutDown(true); +void ClusterControllerApiRpcService::close() { + _closed.store(true); } -void -FNetListener::initRPC() -{ - FRT_ReflectionBuilder rb(_orb.get()); +void ClusterControllerApiRpcService::register_server_methods(SharedRpcResources& rpc_resources) { + FRT_ReflectionBuilder rb(&rpc_resources.supervisor()); - rb.DefineMethod("getnodestate3", "sii", "ss", FRT_METHOD(FNetListener::RPC_getNodeState2), this); + rb.DefineMethod("getnodestate3", "sii", "ss", FRT_METHOD(ClusterControllerApiRpcService::RPC_getNodeState2), this); rb.MethodDesc("Get state of this node"); rb.ParamDesc("nodestate", "Expected state of given node. If correct, the " - "request will be queued on target until it changes. To not give " - "any state use the string 'unknown', enforcing a direct reply."); + "request will be queued on target until it changes. To not give " + "any state use the string 'unknown', enforcing a direct reply."); rb.ParamDesc("timeout", "Timeout of message in milliseconds, set by the state requester"); rb.ReturnDesc("nodestate", "State string for this node"); rb.ReturnDesc("hostinfo", "Information about host this node is running on"); //------------------------------------------------------------------------- - rb.DefineMethod("getnodestate2", "si", "s", FRT_METHOD(FNetListener::RPC_getNodeState2), this); + rb.DefineMethod("getnodestate2", "si", "s", FRT_METHOD(ClusterControllerApiRpcService::RPC_getNodeState2), this); rb.MethodDesc("Get state of this node"); rb.ParamDesc("nodestate", "Expected state of given node. If correct, the " - "request will be queued on target until it changes. To not give " - "any state use the string 'unknown', enforcing a direct reply."); + "request will be queued on target until it changes. To not give " + "any state use the string 'unknown', enforcing a direct reply."); rb.ParamDesc("timeout", "Timeout of message in milliseconds, set by the state requester"); rb.ReturnDesc("nodestate", "State string for this node"); //------------------------------------------------------------------------- - rb.DefineMethod("setsystemstate2", "s", "", FRT_METHOD(FNetListener::RPC_setSystemState2), this); + rb.DefineMethod("setsystemstate2", "s", "", FRT_METHOD(ClusterControllerApiRpcService::RPC_setSystemState2), this); rb.MethodDesc("Set systemstate on this node"); rb.ParamDesc("systemstate", "New systemstate to set"); //------------------------------------------------------------------------- - rb.DefineMethod("setdistributionstates", "bix", "", FRT_METHOD(FNetListener::RPC_setDistributionStates), this); + rb.DefineMethod("setdistributionstates", "bix", "", FRT_METHOD(ClusterControllerApiRpcService::RPC_setDistributionStates), this); rb.MethodDesc("Set distribution states for cluster and bucket spaces"); rb.ParamDesc("compressionType", "Compression type for payload"); rb.ParamDesc("uncompressedSize", "Uncompressed size for payload"); rb.ParamDesc("payload", "Binary Slime format payload"); //------------------------------------------------------------------------- - rb.DefineMethod("activate_cluster_state_version", "i", "i", FRT_METHOD(FNetListener::RPC_activateClusterStateVersion), this); + rb.DefineMethod("activate_cluster_state_version", "i", "i", FRT_METHOD(ClusterControllerApiRpcService::RPC_activateClusterStateVersion), this); rb.MethodDesc("Explicitly activates an already prepared cluster state version"); rb.ParamDesc("activate_version", "Expected cluster state version to activate"); rb.ReturnDesc("actual_version", "Cluster state version that was prepared on the node prior to receiving RPC"); //------------------------------------------------------------------------- - rb.DefineMethod("getcurrenttime", "", "lis", FRT_METHOD(FNetListener::RPC_getCurrentTime), this); + rb.DefineMethod("getcurrenttime", "", "lis", FRT_METHOD(ClusterControllerApiRpcService::RPC_getCurrentTime), this); rb.MethodDesc("Get current time on this node"); rb.ReturnDesc("seconds", "Current time in seconds since epoch"); rb.ReturnDesc("nanoseconds", "additional nanoseconds since epoch"); rb.ReturnDesc("hostname", "Host name"); - //------------------------------------------------------------------------- } - -void -FNetListener::RPC_getCurrentTime(FRT_RPCRequest *req) -{ +// TODO remove? is this used by anyone? +void ClusterControllerApiRpcService::RPC_getCurrentTime(FRT_RPCRequest* req) { if (_closed) { LOG(debug, "Not handling RPC call getCurrentTime() as we have closed"); req->SetError(RPCRequestWrapper::ERR_NODE_SHUTTING_DOWN, "Node shutting down"); @@ -127,19 +89,19 @@ FNetListener::RPC_getCurrentTime(FRT_RPCRequest *req) vespalib::string hostname = vespalib::HostName::get(); req->GetReturn()->AddString(hostname.c_str()); // all handled, will return immediately - return; } -void FNetListener::detach_and_forward_to_enqueuer(std::shared_ptr<api::StorageMessage> cmd, FRT_RPCRequest *req) { +void ClusterControllerApiRpcService::detach_and_forward_to_enqueuer( + std::shared_ptr<api::StorageMessage> cmd, + FRT_RPCRequest* req) +{ // Create a request object to avoid needing a separate transport type cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::make_unique<RPCRequestWrapper>(req))); req->Detach(); - _messageEnqueuer.enqueue(std::move(cmd)); + _message_dispatcher.dispatch_async(std::move(cmd)); } -void -FNetListener::RPC_getNodeState2(FRT_RPCRequest *req) -{ +void ClusterControllerApiRpcService::RPC_getNodeState2(FRT_RPCRequest* req) { if (_closed) { LOG(debug, "Not handling RPC call getNodeState2() as we have closed"); req->SetError(RPCRequestWrapper::ERR_NODE_SHUTTING_DOWN, "Node shutting down"); @@ -147,11 +109,11 @@ FNetListener::RPC_getNodeState2(FRT_RPCRequest *req) } vespalib::string expected(req->GetParams()->GetValue(0)._string._str, - req->GetParams()->GetValue(0)._string._len); + req->GetParams()->GetValue(0)._string._len); - auto cmd(std::make_shared<api::GetNodeStateCommand>(expected != "unknown" - ? std::make_unique<lib::NodeState>(expected) - : std::unique_ptr<lib::NodeState>())); + auto cmd = std::make_shared<api::GetNodeStateCommand>(expected != "unknown" + ? std::make_unique<lib::NodeState>(expected) + : std::unique_ptr<lib::NodeState>()); cmd->setPriority(api::StorageMessage::VERYHIGH); cmd->setTimeout(std::chrono::milliseconds(req->GetParams()->GetValue(1)._intval32)); @@ -161,9 +123,7 @@ FNetListener::RPC_getNodeState2(FRT_RPCRequest *req) detach_and_forward_to_enqueuer(std::move(cmd), req); } -void -FNetListener::RPC_setSystemState2(FRT_RPCRequest *req) -{ +void ClusterControllerApiRpcService::RPC_setSystemState2(FRT_RPCRequest* req) { if (_closed) { LOG(debug, "Not handling RPC call setSystemState2() as we have closed"); req->SetError(RPCRequestWrapper::ERR_NODE_SHUTTING_DOWN, "Node shutting down"); @@ -173,7 +133,7 @@ FNetListener::RPC_setSystemState2(FRT_RPCRequest *req) req->GetParams()->GetValue(0)._string._len); lib::ClusterState systemState(systemStateStr); - auto cmd(std::make_shared<api::SetSystemStateCommand>(lib::ClusterStateBundle(systemState))); + auto cmd = std::make_shared<api::SetSystemStateCommand>(lib::ClusterStateBundle(systemState)); cmd->setPriority(api::StorageMessage::VERYHIGH); detach_and_forward_to_enqueuer(std::move(cmd), req); @@ -183,10 +143,10 @@ namespace { std::shared_ptr<const lib::ClusterStateBundle> decode_bundle_from_params(const FRT_Values& params) { const uint32_t uncompressed_length = params[1]._intval32; - if (uncompressed_length > FNetListener::StateBundleMaxUncompressedSize) { + if (uncompressed_length > ClusterControllerApiRpcService::StateBundleMaxUncompressedSize) { throw std::range_error(vespalib::make_string("RPC ClusterStateBundle uncompressed size (%u) is " "greater than max size (%u)", uncompressed_length, - FNetListener::StateBundleMaxUncompressedSize)); + ClusterControllerApiRpcService::StateBundleMaxUncompressedSize)); } SlimeClusterStateBundleCodec codec; EncodedClusterStateBundle encoded_bundle; @@ -200,7 +160,7 @@ std::shared_ptr<const lib::ClusterStateBundle> decode_bundle_from_params(const F } -void FNetListener::RPC_setDistributionStates(FRT_RPCRequest* req) { +void ClusterControllerApiRpcService::RPC_setDistributionStates(FRT_RPCRequest* req) { if (_closed) { LOG(debug, "Not handling RPC call setDistributionStates() as we have closed"); req->SetError(RPCRequestWrapper::ERR_NODE_SHUTTING_DOWN, "Node shutting down"); @@ -223,7 +183,7 @@ void FNetListener::RPC_setDistributionStates(FRT_RPCRequest* req) { detach_and_forward_to_enqueuer(std::move(cmd), req); } -void FNetListener::RPC_activateClusterStateVersion(FRT_RPCRequest* req) { +void ClusterControllerApiRpcService::RPC_activateClusterStateVersion(FRT_RPCRequest* req) { if (_closed) { LOG(debug, "Not handling RPC call activate_cluster_state_version() as we have closed"); req->SetError(RPCRequestWrapper::ERR_NODE_SHUTTING_DOWN, "Node shutting down"); diff --git a/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h new file mode 100644 index 00000000000..793644194dc --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_controller_api_rpc_service.h @@ -0,0 +1,50 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fnet/frt/invokable.h> +#include <atomic> +#include <memory> + +class FRT_RPCRequest; + +namespace storage { + +class MessageDispatcher; + +namespace api { +class StorageCommand; +class StorageMessage; +class StorageMessageAddress; +class StorageReply; +} + +namespace rpc { + +class SharedRpcResources; + +class ClusterControllerApiRpcService : public FRT_Invokable { + MessageDispatcher& _message_dispatcher; + SharedRpcResources& _rpc_resources; + std::atomic<bool> _closed; +public: + static constexpr uint32_t StateBundleMaxUncompressedSize = 1024 * 1024 * 16; + + ClusterControllerApiRpcService(MessageDispatcher& message_dispatcher, + SharedRpcResources& rpc_resources); + ~ClusterControllerApiRpcService() override; + + void close(); + + void RPC_getNodeState2(FRT_RPCRequest* req); + void RPC_setSystemState2(FRT_RPCRequest* req); + void RPC_getCurrentTime(FRT_RPCRequest* req); + void RPC_setDistributionStates(FRT_RPCRequest* req); + void RPC_activateClusterStateVersion(FRT_RPCRequest* req); +private: + void register_server_methods(SharedRpcResources&); + // TODO factor out as shared functionality + void detach_and_forward_to_enqueuer(std::shared_ptr<api::StorageMessage> cmd, FRT_RPCRequest* req); +}; + +} // rpc +} // storage diff --git a/storage/src/vespa/storage/storageserver/cluster_state_bundle_codec.h b/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h index 41c5db9876d..ea6ef2649d0 100644 --- a/storage/src/vespa/storage/storageserver/cluster_state_bundle_codec.h +++ b/storage/src/vespa/storage/storageserver/rpc/cluster_state_bundle_codec.h @@ -4,9 +4,9 @@ #include "encoded_cluster_state_bundle.h" -namespace storage { +namespace storage::lib { class ClusterStateBundle; } -namespace lib { class ClusterStateBundle; } +namespace storage::rpc { /** * Provides opaque encoding and decoding of ClusterStateBundles for transmission over RPC. diff --git a/storage/src/vespa/storage/storageserver/encoded_cluster_state_bundle.h b/storage/src/vespa/storage/storageserver/rpc/encoded_cluster_state_bundle.h index 6f25a6b67a6..92e66aab378 100644 --- a/storage/src/vespa/storage/storageserver/encoded_cluster_state_bundle.h +++ b/storage/src/vespa/storage/storageserver/rpc/encoded_cluster_state_bundle.h @@ -5,7 +5,7 @@ #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/compressor.h> -namespace storage { +namespace storage::rpc { /** * Contains an opaque encoded (possibly compressed) representation of a ClusterStateBundle. diff --git a/storage/src/vespa/storage/storageserver/rpc/message_codec_provider.cpp b/storage/src/vespa/storage/storageserver/rpc/message_codec_provider.cpp new file mode 100644 index 00000000000..90ea4291a30 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/message_codec_provider.cpp @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "message_codec_provider.h" +#include <vespa/storageapi/mbusprot/protocolserialization7.h> + +namespace storage::rpc { + +WrappedCodec::WrappedCodec(std::shared_ptr<const document::DocumentTypeRepo> doc_type_repo, + std::shared_ptr<const documentapi::LoadTypeSet> load_type_set) + : _doc_type_repo(std::move(doc_type_repo)), + _load_type_set(std::move(load_type_set)), + _codec(std::make_unique<mbusprot::ProtocolSerialization7>(_doc_type_repo, *_load_type_set)) +{ +} + +WrappedCodec::~WrappedCodec() = default; + +MessageCodecProvider::MessageCodecProvider(std::shared_ptr<const document::DocumentTypeRepo> doc_type_repo, + std::shared_ptr<const documentapi::LoadTypeSet> load_type_set) + : _rw_mutex(), + _active_codec(std::make_shared<WrappedCodec>(std::move(doc_type_repo), std::move(load_type_set))) +{ +} + +MessageCodecProvider::~MessageCodecProvider() = default; + +std::shared_ptr<const WrappedCodec> MessageCodecProvider::wrapped_codec() const noexcept { + std::shared_lock r_lock(_rw_mutex); + return _active_codec; +} + +void MessageCodecProvider::update_atomically(std::shared_ptr<const document::DocumentTypeRepo> doc_type_repo, + std::shared_ptr<const documentapi::LoadTypeSet> load_type_set) +{ + std::unique_lock w_lock(_rw_mutex); + _active_codec = std::make_shared<WrappedCodec>(std::move(doc_type_repo), std::move(load_type_set)); +} + +} diff --git a/storage/src/vespa/storage/storageserver/rpc/message_codec_provider.h b/storage/src/vespa/storage/storageserver/rpc/message_codec_provider.h new file mode 100644 index 00000000000..fdadfd6f910 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/message_codec_provider.h @@ -0,0 +1,47 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <shared_mutex> + +namespace document { class DocumentTypeRepo; } +namespace documentapi { class LoadTypeSet; } +namespace storage::mbusprot { class ProtocolSerialization7; } + +namespace storage::rpc { + +class WrappedCodec { + const std::shared_ptr<const document::DocumentTypeRepo> _doc_type_repo; + const std::shared_ptr<const documentapi::LoadTypeSet> _load_type_set; + std::unique_ptr<mbusprot::ProtocolSerialization7> _codec; +public: + WrappedCodec(std::shared_ptr<const document::DocumentTypeRepo> doc_type_repo, + std::shared_ptr<const documentapi::LoadTypeSet> load_type_set); + ~WrappedCodec(); + + [[nodiscard]] const mbusprot::ProtocolSerialization7& codec() const noexcept { return *_codec; } +}; + +/** + * Thread-safe wrapper around a protocol serialization codec and its transitive + * dependencies. Effectively provides support for setting and getting an immutable + * codec snapshot that can be used for RPC (de-)serialization. + */ +class MessageCodecProvider { + // TODO replace with std::atomic<std::shared_ptr<WrappedCodec>> once on a sufficiently new + // C++20 STL that implements the P0718R2 proposal. We expect(tm) an implementation to use + // lock-free compiler-specific 128-bit CAS atomics instead of explicit locks there. + mutable std::shared_mutex _rw_mutex; + std::shared_ptr<WrappedCodec> _active_codec; +public: + MessageCodecProvider(std::shared_ptr<const document::DocumentTypeRepo> doc_type_repo, + std::shared_ptr<const documentapi::LoadTypeSet> load_type_set); + ~MessageCodecProvider(); + + [[nodiscard]] std::shared_ptr<const WrappedCodec> wrapped_codec() const noexcept; + + void update_atomically(std::shared_ptr<const document::DocumentTypeRepo> doc_type_repo, + std::shared_ptr<const documentapi::LoadTypeSet> load_type_set); +}; + +} diff --git a/storage/src/vespa/storage/storageserver/rpc/protobuf/rpc_envelope.proto b/storage/src/vespa/storage/storageserver/rpc/protobuf/rpc_envelope.proto new file mode 100644 index 00000000000..2eab6068bad --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/protobuf/rpc_envelope.proto @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.protobuf; + +message RequestHeader { + uint64 time_remaining_ms = 1; + uint32 trace_level = 2; +} + +message ResponseHeader { + bytes trace_payload = 1; +} diff --git a/storage/src/vespa/storage/storageserver/rpc/rpc_target.cpp b/storage/src/vespa/storage/storageserver/rpc/rpc_target.cpp new file mode 100644 index 00000000000..5fcce1e2a6d --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/rpc_target.cpp @@ -0,0 +1,14 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "rpc_target.h" + +namespace storage::rpc { + +RpcTarget::RpcTarget(std::unique_ptr<WrappedFrtTarget> target, vespalib::stringref spec, uint32_t slobrok_gen) + : _target(std::move(target)), + _spec(spec), + _slobrok_gen(slobrok_gen) +{} + +RpcTarget::~RpcTarget() = default; + +} diff --git a/storage/src/vespa/storage/storageserver/rpc/rpc_target.h b/storage/src/vespa/storage/storageserver/rpc/rpc_target.h new file mode 100644 index 00000000000..0f2dec4269c --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/rpc_target.h @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <cstdint> + +class FRT_Target; + +namespace storage::rpc { + +/** + * Simple wrapper API to access a FRT_Target. + */ +class WrappedFrtTarget { +public: + virtual ~WrappedFrtTarget() = default; + virtual FRT_Target* get() noexcept = 0; + virtual bool is_valid() const noexcept = 0; +}; + +struct RpcTarget { + std::unique_ptr<WrappedFrtTarget> _target; + const vespalib::string _spec; + uint32_t _slobrok_gen; + + RpcTarget(std::unique_ptr<WrappedFrtTarget> target, + vespalib::stringref spec, + uint32_t slobrok_gen); + RpcTarget(const RpcTarget&) = delete; + RpcTarget& operator=(const RpcTarget&) = delete; + RpcTarget(RpcTarget&&) = delete; + RpcTarget& operator=(RpcTarget&&) = delete; + ~RpcTarget(); +}; + +} diff --git a/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h b/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h new file mode 100644 index 00000000000..8411a273dc2 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/rpc_target_factory.h @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace storage::rpc { + +class RpcTarget; + +/** + * Factory for creating instances of RpcTarget based on a connection spec. + */ +class RpcTargetFactory { +public: + virtual ~RpcTargetFactory() = default; + virtual std::unique_ptr<RpcTarget> make_target(const vespalib::string& connection_spec, uint32_t slobrok_gen) const = 0; +}; + +} + diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp new file mode 100644 index 00000000000..d21f32aa623 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp @@ -0,0 +1,121 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "rpc_target.h" +#include "shared_rpc_resources.h" +#include <vespa/fastos/thread.h> +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/fnet/transport.h> +#include <vespa/slobrok/sbregister.h> +#include <vespa/slobrok/sbmirror.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/host_name.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> +#include <chrono> +#include <thread> + +#include <vespa/log/log.h> +LOG_SETUP(".storage.shared_rpc_resources"); + +using namespace std::chrono_literals; + +namespace storage::rpc { + +namespace { + +class WrappedFrtTargetImpl : public WrappedFrtTarget { +private: + FRT_Target* _target; + +public: + WrappedFrtTargetImpl(FRT_Target* target) + : _target(target) + {} + ~WrappedFrtTargetImpl() override { + _target->SubRef(); + } + FRT_Target* get() noexcept override { return _target; } + bool is_valid() const noexcept override { return _target->IsValid(); } +}; + +} + +class SharedRpcResources::RpcTargetFactoryImpl : public RpcTargetFactory { +private: + FRT_Supervisor& _orb; + +public: + RpcTargetFactoryImpl(FRT_Supervisor& orb) + : _orb(orb) + {} + std::unique_ptr<RpcTarget> make_target(const vespalib::string& connection_spec, uint32_t slobrok_gen) const override { + auto* raw_target = _orb.GetTarget(connection_spec.c_str()); + if (raw_target) { + return std::make_unique<RpcTarget> + (std::make_unique<WrappedFrtTargetImpl>(raw_target), connection_spec, slobrok_gen); + } + return std::unique_ptr<RpcTarget>(); + } +}; + +SharedRpcResources::SharedRpcResources(const config::ConfigUri& config_uri, + int rpc_server_port, + size_t rpc_thread_pool_size) + : _thread_pool(std::make_unique<FastOS_ThreadPool>(1024*60)), + _transport(std::make_unique<FNET_Transport>(rpc_thread_pool_size)), + _orb(std::make_unique<FRT_Supervisor>(_transport.get())), + _slobrok_register(std::make_unique<slobrok::api::RegisterAPI>(*_orb, config_uri)), + _slobrok_mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, config_uri)), + _target_factory(std::make_unique<RpcTargetFactoryImpl>(*_orb)), + _hostname(vespalib::HostName::get()), + _rpc_server_port(rpc_server_port), + _shutdown(false) +{ +} + +// TODO make sure init/shutdown is safe for aborted init in comm. mgr. + +SharedRpcResources::~SharedRpcResources() { + if (!_shutdown) { + shutdown(); + } +} + +void SharedRpcResources::start_server_and_register_slobrok(vespalib::stringref my_handle) { + LOG(debug, "Starting main RPC supervisor on port %d with slobrok handle '%s'", + _rpc_server_port, vespalib::string(my_handle).c_str()); + if (!_orb->Listen(_rpc_server_port)) { + throw vespalib::IllegalStateException(vespalib::make_string("Failed to listen to RPC port %d", _rpc_server_port), + VESPA_STRLOC); + } + _transport->Start(_thread_pool.get()); + _slobrok_register->registerName(my_handle); + wait_until_slobrok_is_ready(); + _handle = my_handle; +} + +void SharedRpcResources::wait_until_slobrok_is_ready() { + // TODO look more closely at how mbus does its slobrok business + while (_slobrok_register->busy() || !_slobrok_mirror->ready()) { + // TODO some form of timeout mechanism here, and warning logging to identify SB issues + LOG(debug, "Waiting for Slobrok to become ready"); + std::this_thread::sleep_for(50ms); + } +} + +void SharedRpcResources::shutdown() { + assert(!_shutdown); + _slobrok_register->unregisterName(_handle); + _transport->ShutDown(true); + _shutdown = true; +} + +int SharedRpcResources::listen_port() const noexcept { + return _orb->GetListenPort(); +} + +const RpcTargetFactory& SharedRpcResources::target_factory() const { + return *_target_factory; +} + +} diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h new file mode 100644 index 00000000000..722d1fd3a81 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h @@ -0,0 +1,58 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "rpc_target_factory.h" +#include <vespa/config/subscription/configuri.h> +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +class FastOS_ThreadPool; +class FNET_Transport; +class FRT_Supervisor; + +namespace slobrok::api { +class RegisterAPI; +class MirrorAPI; +} + +namespace storage::rpc { + +class SharedRpcResources { + class RpcTargetFactoryImpl; + std::unique_ptr<FastOS_ThreadPool> _thread_pool; + std::unique_ptr<FNET_Transport> _transport; + std::unique_ptr<FRT_Supervisor> _orb; + std::unique_ptr<slobrok::api::RegisterAPI> _slobrok_register; + std::unique_ptr<slobrok::api::MirrorAPI> _slobrok_mirror; + std::unique_ptr<RpcTargetFactoryImpl> _target_factory; + vespalib::string _hostname; + vespalib::string _handle; + int _rpc_server_port; + bool _shutdown; +public: + SharedRpcResources(const config::ConfigUri& config_uri, int rpc_server_port, size_t rpc_thread_pool_size); + ~SharedRpcResources(); + + FRT_Supervisor& supervisor() noexcept { return *_orb; } + const FRT_Supervisor& supervisor() const noexcept { return *_orb; } + + slobrok::api::RegisterAPI& slobrok_register() noexcept { return *_slobrok_register; } + const slobrok::api::RegisterAPI& slobrok_register() const noexcept { return *_slobrok_register; } + slobrok::api::MirrorAPI& slobrok_mirror() noexcept { return *_slobrok_mirror; } + const slobrok::api::MirrorAPI& slobrok_mirror() const noexcept { return *_slobrok_mirror; } + // To be called after all RPC handlers have been registered. + void start_server_and_register_slobrok(vespalib::stringref my_handle); + + void shutdown(); + [[nodiscard]] int listen_port() const noexcept; // Only valid if server has been started + + // Hostname of host node is running on. + [[nodiscard]] const vespalib::string& hostname() const noexcept { return _hostname; } + + const RpcTargetFactory& target_factory() const; +private: + void wait_until_slobrok_is_ready(); +}; + + +} diff --git a/storage/src/vespa/storage/storageserver/slime_cluster_state_bundle_codec.cpp b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp index 1f854bc724e..0e8a3081aa2 100644 --- a/storage/src/vespa/storage/storageserver/slime_cluster_state_bundle_codec.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp @@ -18,7 +18,7 @@ using vespalib::compression::compress; using vespalib::Memory; using namespace vespalib::slime; -namespace storage { +namespace storage::rpc { // TODO find a suitable home for this class to avoid dupes with rpcsendv2.cpp namespace { diff --git a/storage/src/vespa/storage/storageserver/slime_cluster_state_bundle_codec.h b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.h index 1fb95134059..0c25de5faa5 100644 --- a/storage/src/vespa/storage/storageserver/slime_cluster_state_bundle_codec.h +++ b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.h @@ -5,7 +5,7 @@ #include "cluster_state_bundle_codec.h" #include <memory> -namespace storage { +namespace storage::rpc { /** * Implementation of ClusterStateBundleCodec which uses structured Slime binary encoding diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp new file mode 100644 index 00000000000..68339e9c493 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -0,0 +1,311 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "storage_api_rpc_service.h" +#include "caching_rpc_target_resolver.h" +#include "message_codec_provider.h" +#include "shared_rpc_resources.h" +#include "rpc_envelope.pb.h" +#include <vespa/fnet/frt/supervisor.h> +#include <vespa/fnet/frt/target.h> +#include <vespa/slobrok/sbmirror.h> +#include <vespa/storage/storageserver/communicationmanager.h> +#include <vespa/storage/storageserver/message_dispatcher.h> +#include <vespa/storage/storageserver/rpcrequestwrapper.h> +#include <vespa/storageapi/mbusprot/protocolserialization7.h> +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/util/compressor.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> + +#include <vespa/log/log.h> +LOG_SETUP(".storage.storage_api_rpc_service"); + +using vespalib::compression::CompressionConfig; + +namespace storage::rpc { + +StorageApiRpcService::StorageApiRpcService(MessageDispatcher& message_dispatcher, + SharedRpcResources& rpc_resources, + MessageCodecProvider& message_codec_provider, + const Params& params) + : _message_dispatcher(message_dispatcher), + _rpc_resources(rpc_resources), + _message_codec_provider(message_codec_provider), + _params(params), + _target_resolver(std::make_unique<CachingRpcTargetResolver>(_rpc_resources.slobrok_mirror(), _rpc_resources.target_factory())) +{ + register_server_methods(rpc_resources); +} + +StorageApiRpcService::~StorageApiRpcService() = default; + +StorageApiRpcService::Params::Params() = default; + +StorageApiRpcService::Params::~Params() = default; + +void StorageApiRpcService::register_server_methods(SharedRpcResources& rpc_resources) { + FRT_ReflectionBuilder rb(&rpc_resources.supervisor()); + rb.DefineMethod("storageapi.v1.send", "bixbix", "bixbix", FRT_METHOD(StorageApiRpcService::RPC_rpc_v1_send), this); + rb.MethodDesc("V1 of StorageAPI direct RPC protocol"); + rb.ParamDesc("header_encoding", "0=raw, 6=lz4"); + rb.ParamDesc("header_decoded_size", "Uncompressed header blob size"); + rb.ParamDesc("header_payload", "The message header blob"); + rb.ParamDesc("body_encoding", "0=raw, 6=lz4"); + rb.ParamDesc("body_decoded_size", "Uncompressed body blob size"); + rb.ParamDesc("body_payload", "The message body blob"); + rb.ReturnDesc("header_encoding", "0=raw, 6=lz4"); + rb.ReturnDesc("header_decoded_size", "Uncompressed header blob size"); + rb.ReturnDesc("header_payload", "The reply header blob"); + rb.ReturnDesc("body_encoding", "0=raw, 6=lz4"); + rb.ReturnDesc("body_decoded_size", "Uncompressed body blob size"); + rb.ReturnDesc("body_payload", "The reply body blob"); +} + +void StorageApiRpcService::detach_and_forward_to_enqueuer(std::shared_ptr<api::StorageMessage> cmd, FRT_RPCRequest* req) { + // Create a request object to avoid needing a separate transport type + cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::make_unique<RPCRequestWrapper>(req))); + req->Detach(); + _message_dispatcher.dispatch_sync(std::move(cmd)); +} + +namespace { + +struct SubRefDeleter { + template <typename T> + void operator()(T* v) const noexcept { + v->SubRef(); + } +}; + +template <typename HeaderType> +bool decode_header_from_rpc_params(const FRT_Values& params, HeaderType& hdr) { + const auto compression_type = vespalib::compression::CompressionConfig::toType(params[0]._intval8); + const uint32_t uncompressed_length = params[1]._intval32; + + if (compression_type == vespalib::compression::CompressionConfig::NONE) { + // Fast-path in the common case where request header is not compressed. + return hdr.ParseFromArray(params[2]._data._buf, params[2]._data._len); + } else { + vespalib::DataBuffer uncompressed(params[2]._data._buf, params[2]._data._len); + vespalib::ConstBufferRef blob(params[2]._data._buf, params[2]._data._len); + decompress(compression_type, uncompressed_length, blob, uncompressed, true); + assert(uncompressed_length == uncompressed.getDataLen()); + return hdr.ParseFromArray(uncompressed.getData(), uncompressed.getDataLen()); + } +} + +// Must be done prior to adding payload +template <typename HeaderType> +void encode_header_into_rpc_params(HeaderType& hdr, FRT_Values& params) { + params.AddInt8(vespalib::compression::CompressionConfig::Type::NONE); // TODO when needed + const auto header_size = hdr.ByteSizeLong(); + assert(header_size <= UINT32_MAX); + params.AddInt32(static_cast<uint32_t>(header_size)); + auto* header_buf = reinterpret_cast<uint8_t*>(params.AddData(header_size)); + hdr.SerializeWithCachedSizesToArray(header_buf); +} + +void compress_and_add_payload_to_rpc_params(mbus::BlobRef payload, + FRT_Values& params, + const CompressionConfig& compression_cfg) { + assert(payload.size() <= UINT32_MAX); + vespalib::ConstBufferRef to_compress(payload.data(), payload.size()); + vespalib::DataBuffer buf(vespalib::roundUp2inN(payload.size())); + auto comp_type = compress(compression_cfg, to_compress, buf, false); + assert(buf.getDataLen() <= UINT32_MAX); + + params.AddInt8(comp_type); + params.AddInt32(static_cast<uint32_t>(to_compress.size())); + params.AddData(buf.stealBuffer(), buf.getDataLen()); +} + +} // anon ns + +template <typename MessageType> +void StorageApiRpcService::encode_and_compress_rpc_payload(const MessageType& msg, FRT_Values& params) { + auto wrapped_codec = _message_codec_provider.wrapped_codec(); + auto payload = wrapped_codec->codec().encode(msg); + + compress_and_add_payload_to_rpc_params(payload, params, _params.compression_config); +} + +template <typename PayloadCodecCallback> +void StorageApiRpcService::uncompress_rpc_payload( + const FRT_Values& params, + PayloadCodecCallback payload_callback) +{ + const auto compression_type = vespalib::compression::CompressionConfig::toType(params[3]._intval8); + const uint32_t uncompressed_length = params[4]._intval32; + // TODO fast path if uncompressed? + vespalib::DataBuffer uncompressed(params[5]._data._buf, params[5]._data._len); + vespalib::ConstBufferRef blob(params[5]._data._buf, params[5]._data._len); + decompress(compression_type, uncompressed_length, blob, uncompressed, true); + assert(uncompressed_length == uncompressed.getDataLen()); + assert(uncompressed_length <= UINT32_MAX); + auto wrapped_codec = _message_codec_provider.wrapped_codec(); + + payload_callback(wrapped_codec->codec(), mbus::BlobRef(uncompressed.getData(), uncompressed_length)); +} + +void StorageApiRpcService::RPC_rpc_v1_send(FRT_RPCRequest* req) { + LOG(debug, "Server: received rpc.v1 request"); + // TODO do we need to manually check the parameter/return spec here? + const auto& params = *req->GetParams(); + protobuf::RequestHeader hdr; + if (!decode_header_from_rpc_params(params, hdr)) { + req->SetError(FRTE_RPC_BAD_REQUEST, "Unable to decode RPC request header protobuf"); + return; + } + std::unique_ptr<mbusprot::StorageCommand> cmd; + uint32_t uncompressed_size = 0; + uncompress_rpc_payload(params, [&cmd, &uncompressed_size](auto& codec, auto payload) { + cmd = codec.decodeCommand(payload); + uncompressed_size = static_cast<uint32_t>(payload.size()); + }); + if (cmd && cmd->has_command()) { + auto scmd = cmd->steal_command(); + scmd->setApproxByteSize(uncompressed_size); + scmd->getTrace().setLevel(hdr.trace_level()); + scmd->setTimeout(std::chrono::milliseconds(hdr.time_remaining_ms())); + req->DiscardBlobs(); + detach_and_forward_to_enqueuer(std::move(scmd), req); + } else { + req->SetError(FRTE_RPC_BAD_REQUEST, "Unable to decode RPC request payload"); + } +} + +void StorageApiRpcService::encode_rpc_v1_response(FRT_RPCRequest& request, const api::StorageReply& reply) { + LOG(debug, "Server: encoding rpc.v1 response header and payload"); + auto* ret = request.GetReturn(); + + // TODO skip encoding header altogether if no relevant fields set? + protobuf::ResponseHeader hdr; + if (reply.getTrace().getLevel() > 0) { + hdr.set_trace_payload(reply.getTrace().getRoot().encode()); + } + // TODO consistent naming... + encode_header_into_rpc_params(hdr, *ret); + encode_and_compress_rpc_payload<api::StorageReply>(reply, *ret); +} + +void StorageApiRpcService::send_rpc_v1_request(std::shared_ptr<api::StorageCommand> cmd) { + LOG(debug, "Client: sending rpc.v1 request for message of type %s", cmd->getType().getName().c_str()); + + assert(cmd->getAddress() != nullptr); + auto target = _target_resolver->resolve_rpc_target(*cmd->getAddress()); + if (!target) { + auto reply = cmd->makeReply(); + reply->setResult(make_no_address_for_service_error(*cmd->getAddress())); + // Always dispatch async for synchronously generated replies, or we risk nuking the + // stack if the reply receiver keeps resending synchronously as well. + _message_dispatcher.dispatch_async(std::move(reply)); + return; + } + std::unique_ptr<FRT_RPCRequest, SubRefDeleter> req(_rpc_resources.supervisor().AllocRPCRequest()); + req->SetMethodName("storageapi.v1.send"); + + protobuf::RequestHeader req_hdr; + req_hdr.set_time_remaining_ms(std::chrono::duration_cast<std::chrono::milliseconds>(cmd->getTimeout()).count()); + req_hdr.set_trace_level(cmd->getTrace().getLevel()); + + auto* params = req->GetParams(); + encode_header_into_rpc_params(req_hdr, *params); + encode_and_compress_rpc_payload<api::StorageCommand>(*cmd, *params); + + const auto timeout = cmd->getTimeout(); + // TODO verify it's fine that we alloc this on the request stash and use it this way + auto& req_ctx = req->getStash().create<RpcRequestContext>(std::move(cmd)); + req->SetContext(FNET_Context(&req_ctx)); + + target->_target->get()->InvokeAsync(req.release(), vespalib::to_s(timeout), this); +} + +void StorageApiRpcService::RequestDone(FRT_RPCRequest* raw_req) { + std::unique_ptr<FRT_RPCRequest, SubRefDeleter> req(raw_req); + auto* req_ctx = static_cast<RpcRequestContext*>(req->GetContext()._value.VOIDP); + if (!req->CheckReturnTypes("bixbix")) { + api::ReturnCode error = map_frt_error_to_storage_api_error(*req, *req_ctx); + LOG(debug, "Client: received rpc.v1 error response: %s", error.toString().c_str()); + auto error_reply = req_ctx->_originator_cmd->makeReply(); + error_reply->setResult(std::move(error)); + // TODO needs tracing of received-event! + _message_dispatcher.dispatch_sync(std::move(error_reply)); + return; + } + LOG(debug, "Client: received rpc.v1 OK response"); + + const auto& ret = *req->GetReturn(); + protobuf::ResponseHeader hdr; + if (!decode_header_from_rpc_params(ret, hdr)) { + assert(false); // TODO generate error reply + return; + } + std::unique_ptr<mbusprot::StorageReply> wrapped_reply; + uncompress_rpc_payload(ret, [&wrapped_reply, req_ctx](auto& codec, auto payload) { + wrapped_reply = codec.decodeReply(payload, *req_ctx->_originator_cmd); + }); + // TODO the reply wrapper does lazy deserialization. Can we/should we ever defer? + auto reply = wrapped_reply->getInternalMessage(); // TODO message stealing + assert(reply); + + if (!hdr.trace_payload().empty()) { + req_ctx->_originator_cmd->getTrace().getRoot().addChild(mbus::TraceNode::decode(hdr.trace_payload())); + } + reply->getTrace().swap(req_ctx->_originator_cmd->getTrace()); + + // TODO ensure that no implicit long-lived refs end up pointing into RPC memory...! + req->DiscardBlobs(); + _message_dispatcher.dispatch_sync(std::move(reply)); +} + +api::ReturnCode +StorageApiRpcService::map_frt_error_to_storage_api_error(FRT_RPCRequest& req, + const RpcRequestContext& req_ctx) { + // TODO determine all codes that must be (re)mapped. Current remapping is adapted from RPCSend + const auto& cmd = *req_ctx._originator_cmd; + auto target_service = CachingRpcTargetResolver::address_to_slobrok_id(*cmd.getAddress()); + switch (req.GetErrorCode()) { + case FRTE_RPC_TIMEOUT: + return api::ReturnCode( + static_cast<api::ReturnCode::Result>(mbus::ErrorCode::TIMEOUT), + vespalib::make_string("A timeout occurred while waiting for '%s' (%g seconds expired); %s", + target_service.c_str(), vespalib::to_s(cmd.getTimeout()), req.GetErrorMessage())); + case FRTE_RPC_CONNECTION: + return api::ReturnCode( + static_cast<api::ReturnCode::Result>(mbus::ErrorCode::CONNECTION_ERROR), + vespalib::make_string("A connection error occurred for '%s'; %s", + target_service.c_str(), req.GetErrorMessage())); + default: + return api::ReturnCode( + static_cast<api::ReturnCode::Result>(mbus::ErrorCode::NETWORK_ERROR), + vespalib::make_string("A network error occurred for '%s'; %s", + target_service.c_str(), req.GetErrorMessage())); + } +} + +api::ReturnCode +StorageApiRpcService::make_no_address_for_service_error(const api::StorageMessageAddress& addr) const { + auto error_code = static_cast<api::ReturnCode::Result>(mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE); + auto error_msg = vespalib::make_string( + "The address of service '%s' could not be resolved. It is not currently " + "registered with the Vespa name server. " + "The service must be having problems, or the routing configuration is wrong. " + "Address resolution attempted from host '%s'", + CachingRpcTargetResolver::address_to_slobrok_id(addr).c_str(), + _rpc_resources.hostname().c_str()); + return api::ReturnCode(error_code, std::move(error_msg)); +} + +/* + * Major TODOs: + * - tracing and trace propagation + * - forwards/backwards compatibility + * - what to remap bounced Not Found errors to internally? + * - lifetime semantics of FRT targets vs requests created from them? + * - lifetime of document type/fieldset repos vs messages + * - is repo ref squirreled away into the messages anywhere? + * - everything else! :3 + */ + +} diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h new file mode 100644 index 00000000000..3fca08acc15 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.h @@ -0,0 +1,84 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "rpc_target.h" +#include <vespa/fnet/frt/invokable.h> +#include <vespa/fnet/frt/invoker.h> +#include <vespa/storageapi/messageapi/returncode.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <atomic> +#include <memory> + +class FRT_RPCRequest; +class FRT_Target; + +namespace document { class DocumentTypeRepo; } +namespace documentapi { class LoadTypeSet; } + +namespace storage { + +class MessageDispatcher; + +namespace api { +class StorageCommand; +class StorageMessage; +class StorageMessageAddress; +class StorageReply; +} + +namespace rpc { + +class CachingRpcTargetResolver; +class MessageCodecProvider; +class SharedRpcResources; + +class StorageApiRpcService : public FRT_Invokable, public FRT_IRequestWait { +public: + struct Params { + vespalib::compression::CompressionConfig compression_config; + + Params(); + ~Params(); + }; +private: + MessageDispatcher& _message_dispatcher; + SharedRpcResources& _rpc_resources; + MessageCodecProvider& _message_codec_provider; + const Params _params; + std::unique_ptr<CachingRpcTargetResolver> _target_resolver; +public: + StorageApiRpcService(MessageDispatcher& message_dispatcher, + SharedRpcResources& rpc_resources, + MessageCodecProvider& message_codec_provider, + const Params& params); + ~StorageApiRpcService() override; + + void RPC_rpc_v1_send(FRT_RPCRequest* req); + void encode_rpc_v1_response(FRT_RPCRequest& request, const api::StorageReply& reply); + void send_rpc_v1_request(std::shared_ptr<api::StorageCommand> cmd); +private: + // TODO dedupe + void detach_and_forward_to_enqueuer(std::shared_ptr<api::StorageMessage> cmd, FRT_RPCRequest* req); + + struct RpcRequestContext { + std::shared_ptr<api::StorageCommand> _originator_cmd; + + explicit RpcRequestContext(std::shared_ptr<api::StorageCommand> cmd) + : _originator_cmd(std::move(cmd)) + {} + }; + + void register_server_methods(SharedRpcResources&); + template <typename PayloadCodecCallback> + void uncompress_rpc_payload(const FRT_Values& params, PayloadCodecCallback payload_callback); + template <typename MessageType> + void encode_and_compress_rpc_payload(const MessageType& msg, FRT_Values& params); + void RequestDone(FRT_RPCRequest* request) override; + + api::ReturnCode map_frt_error_to_storage_api_error(FRT_RPCRequest& req, const RpcRequestContext& req_ctx); + api::ReturnCode make_no_address_for_service_error(const api::StorageMessageAddress& addr) const; +}; + +} // rpc +} // storage diff --git a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp index 5f3ff86fe8c..2a083a2d704 100644 --- a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp +++ b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.cpp @@ -7,13 +7,13 @@ namespace storage { RPCRequestWrapper::RPCRequestWrapper(FRT_RPCRequest *req) - : _req(req) + : _req(req) { } RPCRequestWrapper::~RPCRequestWrapper() { - if (_req != 0) { + if (_req) { _req->SetError(ERR_REQUEST_DELETED, "Request deleted without having been replied to"); _req->Return(); } @@ -22,7 +22,7 @@ RPCRequestWrapper::~RPCRequestWrapper() const char * RPCRequestWrapper::getParam() const { - assert(_req != 0); + assert(_req); return _req->GetParams()->GetValue(0)._data._buf; } @@ -30,7 +30,7 @@ RPCRequestWrapper::getParam() const uint32_t RPCRequestWrapper::getParamLen() const { - assert(_req != 0); + assert(_req); return _req->GetParams()->GetValue(0)._data._len; } @@ -38,26 +38,26 @@ RPCRequestWrapper::getParamLen() const void RPCRequestWrapper::returnData(const char *pt, uint32_t len) { - assert(_req != 0); + assert(_req); _req->GetReturn()->AddData(pt, len); _req->Return(); - _req = 0; + _req = nullptr; } void RPCRequestWrapper::returnError(uint32_t errorCode, const char *errorMessage) { - assert(_req != 0); + assert(_req); _req->SetError(errorCode, errorMessage); _req->Return(); - _req = 0; + _req = nullptr; } void RPCRequestWrapper::addReturnString(const char *str, uint32_t len) { - assert(_req != 0); + assert(_req); if (len !=0) { _req->GetReturn()->AddString(str, len); } else { @@ -68,16 +68,16 @@ RPCRequestWrapper::addReturnString(const char *str, uint32_t len) void RPCRequestWrapper::addReturnInt(uint32_t value) { - assert(_req != 0); + assert(_req); _req->GetReturn()->AddInt32(value); } void RPCRequestWrapper::returnRequest() { - assert(_req != 0); + assert(_req); _req->Return(); - _req = 0; + _req = nullptr; } @@ -89,7 +89,7 @@ RPCRequestWrapper::getMethodName() const { void RPCRequestWrapper::discardBlobs() { - if (_req != 0) { + if (_req) { _req->DiscardBlobs(); } } diff --git a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h index 8412e911601..e07163613ea 100644 --- a/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h +++ b/storage/src/vespa/storage/storageserver/rpcrequestwrapper.h @@ -50,6 +50,8 @@ public: **/ void returnError(uint32_t errorCode, const char *errorMessage); + FRT_RPCRequest* raw_request() noexcept { return _req; } + const char *getMethodName() const; void addReturnString(const char *str, uint32_t len=0); void addReturnInt(uint32_t value); @@ -66,7 +68,7 @@ private: RPCRequestWrapper(const RPCRequestWrapper &); RPCRequestWrapper &operator=(const RPCRequestWrapper &); - FRT_RPCRequest *_req; // underlying RPC request + FRT_RPCRequest* _req; // underlying RPC request }; } // namespace storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h index f3499150278..bb7f0308efa 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h @@ -5,21 +5,21 @@ #include "protocolserialization.h" #include <vespa/documentapi/loadtypes/loadtypeset.h> -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { /** * Protocol serialization version that uses Protocol Buffers for all its binary * encoding and decoding. */ -class ProtocolSerialization7 : public ProtocolSerialization { +class ProtocolSerialization7 final : public ProtocolSerialization { const std::shared_ptr<const document::DocumentTypeRepo> _repo; const documentapi::LoadTypeSet& _load_types; public: ProtocolSerialization7(std::shared_ptr<const document::DocumentTypeRepo> repo, const documentapi::LoadTypeSet& load_types); - const document::DocumentTypeRepo& type_repo() const { return *_repo; } + const document::DocumentTypeRepo& type_repo() const noexcept { return *_repo; } + const documentapi::LoadTypeSet& load_type_set() const noexcept { return _load_types; } // Put void onEncode(GBBuf&, const api::PutCommand&) const override; @@ -143,4 +143,3 @@ private: }; } -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h index 26c9ef00752..ef1d5082a04 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h +++ b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h @@ -12,7 +12,7 @@ class StorageCommand : public mbus::Message, public StorageMessage { public: typedef std::unique_ptr<StorageCommand> UP; - StorageCommand(api::StorageCommand::SP); + explicit StorageCommand(api::StorageCommand::SP); const mbus::string & getProtocol() const override { return StorageProtocol::NAME; } uint32_t getType() const override { return _cmd->getType().getId(); } @@ -21,6 +21,9 @@ public: api::StorageMessage::SP getInternalMessage() override { return _cmd; } api::StorageMessage::CSP getInternalMessage() const override { return _cmd; } + bool has_command() const noexcept { return (_cmd.get() != nullptr); } + api::StorageCommand::SP steal_command() { return std::move(_cmd); } + bool hasBucketSequence() const override { return false; } uint8_t priority() const override { diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 93030f699cc..d1bd24f5087 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -105,7 +105,7 @@ const MessageType MessageType::SETBUCKETSTATE_REPLY("SetBucketStateReply", SETBU const MessageType& MessageType::MessageType::get(Id id) { - std::map<Id, MessageType*>::const_iterator it = _codes.find(id); + auto it = _codes.find(id); if (it == _codes.end()) { std::ostringstream ost; ost << "No message type with id " << id << "."; @@ -115,13 +115,13 @@ MessageType::MessageType::get(Id id) } MessageType::MessageType(vespalib::stringref name, Id id, const MessageType* replyOf) - : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) + : _name(name), _id(id), _reply(nullptr), _replyOf(replyOf) { _codes[id] = this; - if (_replyOf != 0) { - assert(_replyOf->_reply == 0); + if (_replyOf) { + assert(_replyOf->_reply == nullptr); // Ugly cast to let initialization work - MessageType& type = const_cast<MessageType&>(*_replyOf); + auto& type = const_cast<MessageType&>(*_replyOf); type._reply = this; } } @@ -144,7 +144,7 @@ StorageMessageAddress::StorageMessageAddress(const mbus::Route& route) _retryEnabled(false), _protocol(DOCUMENT), _cluster(""), - _type(0), + _type(nullptr), _index(0xFFFF) { } @@ -179,7 +179,7 @@ StorageMessageAddress::~StorageMessageAddress() = default; uint16_t StorageMessageAddress::getIndex() const { - if (_type == 0) { + if (!_type) { throw vespalib::IllegalStateException("Cannot retrieve node index out of external address", VESPA_STRLOC); } return _index; @@ -188,7 +188,7 @@ StorageMessageAddress::getIndex() const const lib::NodeType& StorageMessageAddress::getNodeType() const { - if (_type == 0) { + if (!_type) { throw vespalib::IllegalStateException("Cannot retrieve node type out of external address", VESPA_STRLOC); } return *_type; @@ -197,7 +197,7 @@ StorageMessageAddress::getNodeType() const const vespalib::string& StorageMessageAddress::getCluster() const { - if (_type == 0) { + if (!_type) { throw vespalib::IllegalStateException("Cannot retrieve cluster out of external address", VESPA_STRLOC); } return _cluster; @@ -209,7 +209,7 @@ StorageMessageAddress::operator==(const StorageMessageAddress& other) const if (_protocol != other._protocol) return false; if (_retryEnabled != other._retryEnabled) return false; if (_type != other._type) return false; - if (_type != 0) { + if (_type) { if (_cluster != other._cluster) return false; if (_index != other._index) return false; if (_type != other._type) return false; @@ -237,7 +237,7 @@ StorageMessageAddress::print(vespalib::asciistream & out) const if (_retryEnabled) { out << ", retry enabled"; } - if (_type == 0) { + if (!_type) { out << ", " << _route.toString() << ")"; } else { out << ", cluster " << _cluster << ", nodetype " << *_type diff --git a/vespalib/src/tests/stash/stash.cpp b/vespalib/src/tests/stash/stash.cpp index ebf38a1343a..deb4f862dbe 100644 --- a/vespalib/src/tests/stash/stash.cpp +++ b/vespalib/src/tests/stash/stash.cpp @@ -253,9 +253,9 @@ TEST("require that the chunk size can be adjusted") { EXPECT_EQUAL(64000u, stash.get_chunk_size()); } -TEST("require that minimal chunk size is 4096") { - Stash stash(128); - EXPECT_EQUAL(4096u, stash.get_chunk_size()); +TEST("require that minimal chunk size is 128") { + Stash stash(50); + EXPECT_EQUAL(128u, stash.get_chunk_size()); } TEST("require that a stash can be moved by construction") { diff --git a/vespalib/src/vespa/vespalib/util/stash.cpp b/vespalib/src/vespa/vespalib/util/stash.cpp index 31580e871db..daff7b7c575 100644 --- a/vespalib/src/vespa/vespalib/util/stash.cpp +++ b/vespalib/src/vespa/vespalib/util/stash.cpp @@ -63,7 +63,7 @@ Stash::do_alloc(size_t size) Stash::Stash(size_t chunk_size) noexcept : _chunks(nullptr), _cleanup(nullptr), - _chunk_size(std::max(size_t(4096), chunk_size)) + _chunk_size(std::max(size_t(128), chunk_size)) { } diff --git a/vespalog/src/logger/runserver.cpp b/vespalog/src/logger/runserver.cpp index 39e4f57418a..7b86eba31aa 100644 --- a/vespalog/src/logger/runserver.cpp +++ b/vespalog/src/logger/runserver.cpp @@ -44,8 +44,13 @@ time_t steady_time() { return duration_cast<seconds>(steady_clock::now().time_since_epoch()).count(); } +bool whole_seconds(int cnt, int secs) { + cnt %= (secs * 10); + return cnt == 0; } +} // namespace + class PidFile { private: @@ -387,20 +392,36 @@ int main(int argc, char *argv[]) if (system(killcmd) != 0) { fprintf(stderr, "WARNING: stop command '%s' had some problem\n", killcmd); } + fflush(stdout); } else { fprintf(stdout, "%s was running with pid %d, sending SIGTERM\n", service, pid); if (kill(pid, SIGTERM) != 0) { fprintf(stderr, "could not signal %d: %s\n", pid, strerror(errno)); + killpg(pid, SIGTERM); return 1; } } fprintf(stdout, "Waiting for exit (up to 15 minutes)\n"); - for (int cnt(0); cnt < 86400; cnt++) { + fflush(stdout); + const int one_day = 24 * 60 * 60 * 10; + const int twelve_minutes = 12 * 60 * 10; + const int fifteen_minutes = 15 * 60 * 10; + for (int cnt(0); cnt < one_day; cnt++) { usleep(100000); // wait 0.1 seconds - if ((cnt > 7200) && (cnt % 100 == 0)) { + if ((cnt < twelve_minutes) && (kill(pid, 0) != 0)) { + if (killpg(pid, SIGTERM) == 0) { + fprintf(stdout, " %s exited, terminating strays in its progress groups\n", service); + fflush(stdout); + } + cnt = twelve_minutes; + } + if ((cnt > twelve_minutes) && whole_seconds(cnt, 10)) { + fprintf(stdout, " %s or its children not stopping: sending SIGTERM to process group %d\n", + service, pid); killpg(pid, SIGTERM); + fflush(stdout); } if (killpg(pid, 0) == 0) { if (cnt%10 == 0) { @@ -408,12 +429,14 @@ int main(int argc, char *argv[]) fflush(stdout); } } else { - fprintf(stdout, "DONE\n"); + fprintf(stdout, " DONE\n"); + fflush(stdout); break; } - if (cnt == 9000) { - printf("\ngiving up, sending KILL signal\n"); + if ((cnt >= fifteen_minutes) && whole_seconds(cnt, 5)) { + printf(" giving up, sending KILL signal\n"); killpg(pid, SIGKILL); + fflush(stdout); } } } else { |